chore(release): publish new version

- @standardnotes/home-server@1.16.45
 - @standardnotes/revisions-server@1.43.2
 - @standardnotes/syncing-server@1.115.2

.github/workflows/common-deploy.yml

@@ -38,10 +38,6 @@ jobs:
       run: |
         jq '(.containerDefinitions[] | select(.name=="${{ inputs.service_name }}-prod") | .environment[] | select(.name=="VERSION")).value = "${{ github.sha }}"' task-definition.json > tmp.json && mv tmp.json task-definition.json
 
-    - name: Fill in the new package version in the Amazon ECS task definition
-      run: |
-        jq '(.containerDefinitions[] | select(.name=="${{ inputs.service_name }}-prod") | .environment[] | select(.name=="npm_package_version")).value = "${{ github.sha }}"' task-definition.json > tmp.json && mv tmp.json task-definition.json
-
     - name: Fill in the new image ID in the Amazon ECS task definition
       id: task-def-prod
       uses: aws-actions/amazon-ecs-render-task-definition@v1

packages/analytics/CHANGELOG.md

@@ -3,6 +3,30 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [2.28.8](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.7...@standardnotes/analytics@2.28.8) (2023-10-10)
+
+**Note:** Version bump only for package @standardnotes/analytics
+
+## [2.28.7](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.6...@standardnotes/analytics@2.28.7) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/analytics
+
+## [2.28.6](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.5...@standardnotes/analytics@2.28.6) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/analytics
+
+## [2.28.5](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.4...@standardnotes/analytics@2.28.5) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/analytics
+
+## [2.28.4](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.3...@standardnotes/analytics@2.28.4) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/analytics
+
+## [2.28.3](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.2...@standardnotes/analytics@2.28.3) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/analytics
+
 ## [2.28.2](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.1...@standardnotes/analytics@2.28.2) (2023-10-09)
 
 ### Bug Fixes

packages/analytics/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/analytics",
-  "version": "2.28.2",
+  "version": "2.28.8",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/api-gateway/CHANGELOG.md

@@ -3,6 +3,32 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.78.5](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.78.4...@standardnotes/api-gateway@1.78.5) (2023-10-10)
+
+**Note:** Version bump only for package @standardnotes/api-gateway
+
+## [1.78.4](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.78.3...@standardnotes/api-gateway@1.78.4) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/api-gateway
+
+## [1.78.3](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.78.2...@standardnotes/api-gateway@1.78.3) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/api-gateway
+
+## [1.78.2](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.78.1...@standardnotes/api-gateway@1.78.2) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/api-gateway
+
+## [1.78.1](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.78.0...@standardnotes/api-gateway@1.78.1) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/api-gateway
+
+# [1.78.0](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.77.2...@standardnotes/api-gateway@1.78.0) (2023-10-09)
+
+### Features
+
+* add opentelemetry to all services ([5e930d0](https://github.com/standardnotes/api-gateway/commit/5e930d08eb60a0da800081342315e7edaf130951))
+
 ## [1.77.2](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.77.1...@standardnotes/api-gateway@1.77.2) (2023-10-09)
 
 ### Bug Fixes

packages/api-gateway/bin/server.ts

@@ -36,6 +36,7 @@ import { InversifyExpressServer } from 'inversify-express-utils'
 import { ContainerConfigLoader } from '../src/Bootstrap/Container'
 import { TYPES } from '../src/Bootstrap/Types'
 import { Env } from '../src/Bootstrap/Env'
+import { OpenTelemetrySDKInterface } from '@standardnotes/domain-events-infra'
 
 const container = new ContainerConfigLoader()
 void container.load().then((container) => {
@@ -104,6 +105,11 @@ void container.load().then((container) => {
 
   const serverInstance = server.build()
 
+  if (!container.get<boolean>(TYPES.ApiGateway_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)) {
+    const openTelemetrySDK = container.get<OpenTelemetrySDKInterface>(TYPES.ApiGateway_OpenTelemetrySDK)
+    openTelemetrySDK.start()
+  }
+
   serverInstance.listen(env.get('PORT'))
 
   logger.info(`Server started on port ${process.env.PORT}`)

packages/api-gateway/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/api-gateway",
-  "version": "1.77.2",
+  "version": "1.78.5",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/api-gateway/src/Bootstrap/Container.ts

@@ -16,12 +16,13 @@ import { RedisCrossServiceTokenCache } from '../Infra/Redis/RedisCrossServiceTok
 import { WebSocketAuthMiddleware } from '../Controller/WebSocketAuthMiddleware'
 import { InMemoryCrossServiceTokenCache } from '../Infra/InMemory/InMemoryCrossServiceTokenCache'
 import { DirectCallServiceProxy } from '../Service/Proxy/DirectCallServiceProxy'
-import { ServiceContainerInterface } from '@standardnotes/domain-core'
+import { ServiceContainerInterface, ServiceIdentifier } from '@standardnotes/domain-core'
 import { EndpointResolverInterface } from '../Service/Resolver/EndpointResolverInterface'
 import { EndpointResolver } from '../Service/Resolver/EndpointResolver'
 import { RequiredCrossServiceTokenMiddleware } from '../Controller/RequiredCrossServiceTokenMiddleware'
 import { OptionalCrossServiceTokenMiddleware } from '../Controller/OptionalCrossServiceTokenMiddleware'
 import { Transform } from 'stream'
+import { OpenTelemetrySDK, OpenTelemetrySDKInterface } from '@standardnotes/domain-events-infra'
 
 export class ContainerConfigLoader {
   async load(configuration?: {
@@ -35,8 +36,20 @@ export class ContainerConfigLoader {
     const container = new Container()
 
     const isConfiguredForHomeServer = env.get('MODE', true) === 'home-server'
+    const isConfiguredForSelfHosting = env.get('MODE', true) === 'self-hosted'
+    const isConfiguredForHomeServerOrSelfHosting = isConfiguredForHomeServer || isConfiguredForSelfHosting
     const isConfiguredForInMemoryCache = env.get('CACHE_TYPE', true) === 'memory'
 
+    container
+      .bind<boolean>(TYPES.ApiGateway_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)
+      .toConstantValue(isConfiguredForHomeServerOrSelfHosting)
+
+    if (!isConfiguredForHomeServerOrSelfHosting) {
+      container
+        .bind<OpenTelemetrySDKInterface>(TYPES.ApiGateway_OpenTelemetrySDK)
+        .toConstantValue(new OpenTelemetrySDK(ServiceIdentifier.NAMES.ApiGateway))
+    }
+
     const winstonFormatters = [winston.format.splat(), winston.format.json()]
     if (env.get('NEW_RELIC_ENABLED', true) === 'true') {
       await import('newrelic')

packages/api-gateway/src/Bootstrap/Types.ts

@@ -15,6 +15,9 @@ export const TYPES = {
   ApiGateway_VERSION: Symbol.for('ApiGateway_VERSION'),
   ApiGateway_CROSS_SERVICE_TOKEN_CACHE_TTL: Symbol.for('ApiGateway_CROSS_SERVICE_TOKEN_CACHE_TTL'),
   ApiGateway_IS_CONFIGURED_FOR_HOME_SERVER: Symbol.for('ApiGateway_IS_CONFIGURED_FOR_HOME_SERVER'),
+  ApiGateway_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING: Symbol.for(
+    'ApiGateway_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING',
+  ),
   // Middleware
   ApiGateway_RequiredCrossServiceTokenMiddleware: Symbol.for('ApiGateway_RequiredCrossServiceTokenMiddleware'),
   ApiGateway_OptionalCrossServiceTokenMiddleware: Symbol.for('ApiGateway_OptionalCrossServiceTokenMiddleware'),
@@ -25,4 +28,5 @@ export const TYPES = {
   ApiGateway_CrossServiceTokenCache: Symbol.for('ApiGateway_CrossServiceTokenCache'),
   ApiGateway_Timer: Symbol.for('ApiGateway_Timer'),
   ApiGateway_EndpointResolver: Symbol.for('ApiGateway_EndpointResolver'),
+  ApiGateway_OpenTelemetrySDK: Symbol.for('ApiGateway_OpenTelemetrySDK'),
 }

packages/auth/CHANGELOG.md

@@ -3,6 +3,42 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [1.155.0](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.154.3...@standardnotes/auth-server@1.155.0) (2023-10-10)
+
+### Features
+
+* add more logs to transition ([783fd9e](https://github.com/standardnotes/server/commit/783fd9e2c6350a7f0f1e8d009d01a3328564aca0))
+
+## [1.154.3](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.154.2...@standardnotes/auth-server@1.154.3) (2023-10-09)
+
+### Reverts
+
+* Revert "Revert "fix: setting parent span on workers"" ([76ae6f5](https://github.com/standardnotes/server/commit/76ae6f5a882a82ab5f635452e3bc7b2b16709531))
+
+## [1.154.2](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.154.1...@standardnotes/auth-server@1.154.2) (2023-10-09)
+
+### Reverts
+
+* Revert "fix: setting parent span on workers" ([3fc07a5](https://github.com/standardnotes/server/commit/3fc07a5b60c26b583efd88e8a80d4c4321e71efb))
+
+## [1.154.1](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.154.0...@standardnotes/auth-server@1.154.1) (2023-10-09)
+
+### Bug Fixes
+
+* setting parent span on workers ([1c54d18](https://github.com/standardnotes/server/commit/1c54d18c3ca75353701ba921492a5ecfaa2e3572))
+
+# [1.154.0](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.153.0...@standardnotes/auth-server@1.154.0) (2023-10-09)
+
+### Features
+
+* add custom tracing on workers ([65ced2c](https://github.com/standardnotes/server/commit/65ced2cc7b0686dc8af5cdad499412fc8fd29d1d))
+
+# [1.153.0](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.152.2...@standardnotes/auth-server@1.153.0) (2023-10-09)
+
+### Features
+
+* add opentelemetry to all services ([5e930d0](https://github.com/standardnotes/server/commit/5e930d08eb60a0da800081342315e7edaf130951))
+
 ## [1.152.2](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.152.1...@standardnotes/auth-server@1.152.2) (2023-10-09)
 
 ### Bug Fixes

packages/auth/bin/server.ts

@@ -29,6 +29,7 @@ import { InversifyExpressServer } from 'inversify-express-utils'
 import { ContainerConfigLoader } from '../src/Bootstrap/Container'
 import TYPES from '../src/Bootstrap/Types'
 import { Env } from '../src/Bootstrap/Env'
+import { OpenTelemetrySDKInterface } from '@standardnotes/domain-events-infra'
 
 const container = new ContainerConfigLoader()
 void container.load().then((container) => {
@@ -66,6 +67,11 @@ void container.load().then((container) => {
 
   const serverInstance = server.build()
 
+  if (!container.get<boolean>(TYPES.Auth_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)) {
+    const openTelemetrySDK = container.get<OpenTelemetrySDKInterface>(TYPES.Auth_OpenTelemetrySDK)
+    openTelemetrySDK.start()
+  }
+
   serverInstance.listen(env.get('PORT'))
 
   logger.info(`Server started on port ${process.env.PORT}`)

packages/auth/bin/transition.ts

@@ -17,6 +17,7 @@ import { RoleName, TransitionStatus } from '@standardnotes/domain-core'
 const inputArgs = process.argv.slice(2)
 const startDateString = inputArgs[0]
 const endDateString = inputArgs[1]
+const forceRunParam = inputArgs[2]
 
 const requestTransition = async (
   transitionStatusRepository: TransitionStatusRepositoryInterface,
@@ -38,6 +39,7 @@ const requestTransition = async (
   )
 
   let usersTriggered = 0
+  const forceRun = forceRunParam === 'true'
   for (const user of users) {
     const itemsTransitionStatus = await transitionStatusRepository.getStatus(user.uuid, 'items')
     const revisionsTransitionStatus = await transitionStatusRepository.getStatus(user.uuid, 'revisions')
@@ -55,7 +57,11 @@ const requestTransition = async (
 
     let wasTransitionRequested = false
 
-    if (itemsTransitionStatus === null || itemsTransitionStatus.value === TransitionStatus.STATUSES.Failed) {
+    if (
+      itemsTransitionStatus === null ||
+      itemsTransitionStatus.value === TransitionStatus.STATUSES.Failed ||
+      (itemsTransitionStatus.value === TransitionStatus.STATUSES.InProgress && forceRun)
+    ) {
       wasTransitionRequested = true
       await transitionStatusRepository.remove(user.uuid, 'items')
 
@@ -68,7 +74,11 @@ const requestTransition = async (
       )
     }
 
-    if (revisionsTransitionStatus === null || revisionsTransitionStatus.value === TransitionStatus.STATUSES.Failed) {
+    if (
+      revisionsTransitionStatus === null ||
+      revisionsTransitionStatus.value === TransitionStatus.STATUSES.Failed ||
+      (revisionsTransitionStatus.value === TransitionStatus.STATUSES.InProgress && forceRun)
+    ) {
       wasTransitionRequested = true
       await transitionStatusRepository.remove(user.uuid, 'revisions')
 

packages/auth/bin/worker.ts

@@ -8,6 +8,7 @@ import { Env } from '../src/Bootstrap/Env'
 import { DomainEventSubscriberFactoryInterface } from '@standardnotes/domain-events'
 import * as dayjs from 'dayjs'
 import * as utc from 'dayjs/plugin/utc'
+import { OpenTelemetrySDKInterface } from '@standardnotes/domain-events-infra'
 
 const container = new ContainerConfigLoader('worker')
 void container.load().then((container) => {
@@ -20,6 +21,11 @@ void container.load().then((container) => {
 
   logger.info('Starting worker...')
 
+  if (!container.get<boolean>(TYPES.Auth_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)) {
+    const openTelemetrySDK = container.get<OpenTelemetrySDKInterface>(TYPES.Auth_OpenTelemetrySDK)
+    openTelemetrySDK.start()
+  }
+
   const subscriberFactory: DomainEventSubscriberFactoryInterface = container.get(
     TYPES.Auth_DomainEventSubscriberFactory,
   )

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/auth-server",
-  "version": "1.152.2",
+  "version": "1.155.0",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/auth/src/Bootstrap/Container.ts

@@ -89,9 +89,12 @@ import { ExtensionKeyGrantedEventHandler } from '../Domain/Handler/ExtensionKeyG
 import {
   DirectCallDomainEventPublisher,
   DirectCallEventMessageHandler,
+  OpenTelemetrySDK,
+  OpenTelemetrySDKInterface,
   SNSDomainEventPublisher,
   SQSDomainEventSubscriberFactory,
   SQSEventMessageHandler,
+  SQSOpenTelemetryEventMessageHandler,
 } from '@standardnotes/domain-events-infra'
 import { GetUserSubscription } from '../Domain/UseCase/GetUserSubscription/GetUserSubscription'
 import { ChangeCredentials } from '../Domain/UseCase/ChangeCredentials/ChangeCredentials'
@@ -188,6 +191,7 @@ import {
   ControllerContainer,
   ControllerContainerInterface,
   MapperInterface,
+  ServiceIdentifier,
   SharedVaultUser,
 } from '@standardnotes/domain-core'
 import { SessionTracePersistenceMapper } from '../Mapping/SessionTracePersistenceMapper'
@@ -319,8 +323,24 @@ export class ContainerConfigLoader {
     logger.debug('Database initialized')
 
     const isConfiguredForHomeServer = env.get('MODE', true) === 'home-server'
+    const isConfiguredForSelfHosting = env.get('MODE', true) === 'self-hosted'
+    const isConfiguredForHomeServerOrSelfHosting = isConfiguredForHomeServer || isConfiguredForSelfHosting
     const isConfiguredForInMemoryCache = env.get('CACHE_TYPE', true) === 'memory'
 
+    container
+      .bind<boolean>(TYPES.Auth_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)
+      .toConstantValue(isConfiguredForHomeServerOrSelfHosting)
+
+    if (!isConfiguredForHomeServerOrSelfHosting) {
+      container
+        .bind<OpenTelemetrySDKInterface>(TYPES.Auth_OpenTelemetrySDK)
+        .toConstantValue(
+          new OpenTelemetrySDK(
+            this.mode === 'server' ? ServiceIdentifier.NAMES.Auth : ServiceIdentifier.NAMES.AuthWorker,
+          ),
+        )
+    }
+
     if (!isConfiguredForInMemoryCache) {
       const redisUrl = env.get('REDIS_URL')
       const isRedisInClusterMode = redisUrl.indexOf(',') > 0
@@ -1223,7 +1243,15 @@ export class ContainerConfigLoader {
     } else {
       container
         .bind<DomainEventMessageHandlerInterface>(TYPES.Auth_DomainEventMessageHandler)
-        .toConstantValue(new SQSEventMessageHandler(eventHandlers, container.get(TYPES.Auth_Logger)))
+        .toConstantValue(
+          isConfiguredForHomeServerOrSelfHosting
+            ? new SQSEventMessageHandler(eventHandlers, container.get(TYPES.Auth_Logger))
+            : new SQSOpenTelemetryEventMessageHandler(
+                ServiceIdentifier.NAMES.AuthWorker,
+                eventHandlers,
+                container.get(TYPES.Auth_Logger),
+              ),
+        )
 
       container
         .bind<DomainEventSubscriberFactoryInterface>(TYPES.Auth_DomainEventSubscriberFactory)

packages/auth/src/Bootstrap/Types.ts

@@ -3,6 +3,7 @@ const TYPES = {
   Auth_Redis: Symbol.for('Auth_Redis'),
   Auth_SNS: Symbol.for('Auth_SNS'),
   Auth_SQS: Symbol.for('Auth_SQS'),
+  Auth_OpenTelemetrySDK: Symbol.for('Auth_OpenTelemetrySDK'),
   // Mapping
   Auth_SessionTracePersistenceMapper: Symbol.for('Auth_SessionTracePersistenceMapper'),
   Auth_AuthenticatorChallengePersistenceMapper: Symbol.for('Auth_AuthenticatorChallengePersistenceMapper'),
@@ -105,6 +106,7 @@ const TYPES = {
   Auth_U2F_EXPECTED_ORIGIN: Symbol.for('Auth_U2F_EXPECTED_ORIGIN'),
   Auth_U2F_REQUIRE_USER_VERIFICATION: Symbol.for('Auth_U2F_REQUIRE_USER_VERIFICATION'),
   Auth_READONLY_USERS: Symbol.for('Auth_READONLY_USERS'),
+  Auth_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING: Symbol.for('Auth_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING'),
   // use cases
   Auth_AuthenticateUser: Symbol.for('Auth_AuthenticateUser'),
   Auth_AuthenticateRequest: Symbol.for('Auth_AuthenticateRequest'),

packages/domain-events-infra/CHANGELOG.md

@@ -3,6 +3,49 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.16.4](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.16.3...@standardnotes/domain-events-infra@1.16.4) (2023-10-10)
+
+### Bug Fixes
+
+* **syncing-server:** add option to define otel ratio ([c021bb3](https://github.com/standardnotes/server/commit/c021bb3d7ca90179292e7c75f5a84bf2b941ce86))
+
+## [1.16.3](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.16.2...@standardnotes/domain-events-infra@1.16.3) (2023-10-09)
+
+### Bug Fixes
+
+* **domain-events-infra:** add parent context on internal span ([099c6e1](https://github.com/standardnotes/server/commit/099c6e10c69b6b81006f59bcdb35325b545ab39c))
+
+### Reverts
+
+* Revert "Revert "fix: setting parent span on workers"" ([76ae6f5](https://github.com/standardnotes/server/commit/76ae6f5a882a82ab5f635452e3bc7b2b16709531))
+
+## [1.16.2](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.16.1...@standardnotes/domain-events-infra@1.16.2) (2023-10-09)
+
+### Reverts
+
+* Revert "fix: setting parent span on workers" ([3fc07a5](https://github.com/standardnotes/server/commit/3fc07a5b60c26b583efd88e8a80d4c4321e71efb))
+
+## [1.16.1](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.16.0...@standardnotes/domain-events-infra@1.16.1) (2023-10-09)
+
+### Bug Fixes
+
+* setting parent span on workers ([1c54d18](https://github.com/standardnotes/server/commit/1c54d18c3ca75353701ba921492a5ecfaa2e3572))
+
+# [1.16.0](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.15.0...@standardnotes/domain-events-infra@1.16.0) (2023-10-09)
+
+### Features
+
+* add custom tracing on workers ([65ced2c](https://github.com/standardnotes/server/commit/65ced2cc7b0686dc8af5cdad499412fc8fd29d1d))
+* **domain-events-infra:** add express instrumentation ([b47c80c](https://github.com/standardnotes/server/commit/b47c80cccd23af5d2ad375ef535a38f57362b8d0))
+* **domain-events-infra:** add ioredis instrumentation ([b72e515](https://github.com/standardnotes/server/commit/b72e515931f5ea874494fa88be7dcee7ad693b3f))
+* **domain-events-infra:** add winston instrumentation ([65fcc65](https://github.com/standardnotes/server/commit/65fcc657a77790c7f1b9b4a872ff59152e56bbc7))
+
+# [1.15.0](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.14.9...@standardnotes/domain-events-infra@1.15.0) (2023-10-09)
+
+### Features
+
+* add opentelemetry to all services ([5e930d0](https://github.com/standardnotes/server/commit/5e930d08eb60a0da800081342315e7edaf130951))
+
 ## [1.14.9](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.14.8...@standardnotes/domain-events-infra@1.14.9) (2023-10-09)
 
 ### Bug Fixes

packages/domain-events-infra/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/domain-events-infra",
-  "version": "1.14.9",
+  "version": "1.16.4",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },
@@ -26,8 +26,22 @@
   "dependencies": {
     "@aws-sdk/client-sns": "^3.332.0",
     "@aws-sdk/client-sqs": "^3.332.0",
+    "@opentelemetry/api": "^1.6.0",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.43.0",
+    "@opentelemetry/exporter-trace-otlp-grpc": "^0.43.0",
+    "@opentelemetry/id-generator-aws-xray": "^1.2.1",
+    "@opentelemetry/instrumentation-aws-sdk": "^0.36.0",
+    "@opentelemetry/instrumentation-express": "^0.33.1",
+    "@opentelemetry/instrumentation-http": "^0.43.0",
+    "@opentelemetry/instrumentation-ioredis": "^0.35.1",
+    "@opentelemetry/instrumentation-winston": "^0.32.1",
+    "@opentelemetry/propagator-aws-xray": "^1.3.1",
+    "@opentelemetry/resource-detector-aws": "^1.3.1",
+    "@opentelemetry/sdk-node": "^0.43.0",
+    "@opentelemetry/semantic-conventions": "^1.17.0",
     "@standardnotes/domain-events": "workspace:*",
     "ioredis": "^5.2.4",
+    "opentelemetry-instrumentation-typeorm": "^0.39.1",
     "reflect-metadata": "^0.1.13",
     "sqs-consumer": "^7.3.0",
     "winston": "^3.8.1"

packages/domain-events-infra/src/Infra/OpenTelemetry/OpenTelemetrySDK.ts

@@ -0,0 +1,81 @@
+import * as OpenTelemetrySDKNode from '@opentelemetry/sdk-node'
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
+import { AWSXRayIdGenerator } from '@opentelemetry/id-generator-aws-xray'
+import * as AwsResourceDetectors from '@opentelemetry/resource-detector-aws'
+import { TypeormInstrumentation } from 'opentelemetry-instrumentation-typeorm'
+import { AWSXRayPropagator } from '@opentelemetry/propagator-aws-xray'
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
+import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto'
+import { WinstonInstrumentation } from '@opentelemetry/instrumentation-winston'
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'
+
+import { OpenTelemetrySDKInterface } from './OpenTelemetrySDKInterface'
+import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express'
+
+export class OpenTelemetrySDK implements OpenTelemetrySDKInterface {
+  private declare sdk: OpenTelemetrySDKNode.NodeSDK
+
+  constructor(
+    private serviceName: string,
+    private spanRatio?: number,
+  ) {
+    this.build()
+  }
+
+  build(): void {
+    const otResource = OpenTelemetrySDKNode.resources.Resource.default().merge(
+      new OpenTelemetrySDKNode.resources.Resource({
+        [SemanticResourceAttributes.SERVICE_NAME]: this.serviceName,
+      }),
+    )
+    const traceExporter = new OTLPTraceExporter()
+    const spanProcessor = new OpenTelemetrySDKNode.tracing.BatchSpanProcessor(traceExporter)
+    const metricReader = new OpenTelemetrySDKNode.metrics.PeriodicExportingMetricReader({
+      exportIntervalMillis: 1_000,
+      exporter: new OTLPMetricExporter(),
+    })
+
+    const serviceName = this.serviceName
+    const winstonInstrumentation = new WinstonInstrumentation({
+      logHook: (_span, record) => {
+        record['resource.service.name'] = serviceName
+      },
+    })
+
+    const ratio = this.spanRatio ?? 0.01
+
+    this.sdk = new OpenTelemetrySDKNode.NodeSDK({
+      sampler: new OpenTelemetrySDKNode.tracing.TraceIdRatioBasedSampler(ratio),
+      textMapPropagator: new AWSXRayPropagator(),
+      instrumentations: [
+        new HttpInstrumentation({
+          ignoreIncomingRequestHook: (request) => {
+            const isHealthCheckUrl = !!request.url?.match(/\/healthcheck/)
+
+            return isHealthCheckUrl
+          },
+        }),
+        new ExpressInstrumentation(),
+        new AwsInstrumentation({
+          suppressInternalInstrumentation: true,
+        }),
+        new TypeormInstrumentation(),
+        winstonInstrumentation,
+        new IORedisInstrumentation(),
+      ],
+      metricReader: metricReader,
+      resource: otResource,
+      spanProcessor: spanProcessor,
+      traceExporter: traceExporter,
+      idGenerator: new AWSXRayIdGenerator(),
+      autoDetectResources: true,
+      resourceDetectors: [AwsResourceDetectors.awsEcsDetector],
+    })
+  }
+
+  start(): void {
+    this.sdk.start()
+  }
+}

packages/domain-events-infra/src/Infra/OpenTelemetry/OpenTelemetrySDKInterface.ts

@@ -0,0 +1,3 @@
+export interface OpenTelemetrySDKInterface {
+  start(): void
+}

packages/domain-events-infra/src/Infra/SQS/SQSOpenTelemetryEventMessageHandler.ts

@@ -0,0 +1,69 @@
+import { Logger } from 'winston'
+import * as zlib from 'zlib'
+import * as OpenTelemetryApi from '@opentelemetry/api'
+import {
+  DomainEventHandlerInterface,
+  DomainEventInterface,
+  DomainEventMessageHandlerInterface,
+} from '@standardnotes/domain-events'
+
+export class SQSOpenTelemetryEventMessageHandler implements DomainEventMessageHandlerInterface {
+  private currentSpan: OpenTelemetryApi.Span | undefined
+  private internalSpan: OpenTelemetryApi.Span | undefined
+
+  constructor(
+    private serviceName: string,
+    private handlers: Map<string, DomainEventHandlerInterface>,
+    private logger: Logger,
+  ) {}
+
+  async handleMessage(message: string): Promise<void> {
+    const messageParsed = JSON.parse(message)
+
+    const domainEventJson = zlib.unzipSync(Buffer.from(messageParsed.Message, 'base64')).toString()
+
+    const domainEvent: DomainEventInterface = JSON.parse(domainEventJson)
+
+    domainEvent.createdAt = new Date(domainEvent.createdAt)
+
+    const handler = this.handlers.get(domainEvent.type)
+    if (!handler) {
+      this.logger.debug(`Event handler for event type ${domainEvent.type} does not exist`)
+
+      return
+    }
+
+    this.logger.debug(`Received event: ${domainEvent.type}`)
+
+    const tracer = OpenTelemetryApi.trace.getTracer('sqs-handler')
+
+    this.currentSpan = tracer.startSpan(this.serviceName, { kind: OpenTelemetryApi.SpanKind.CONSUMER })
+    const ctx = OpenTelemetryApi.trace.setSpan(OpenTelemetryApi.context.active(), this.currentSpan)
+
+    this.internalSpan = tracer.startSpan(domainEvent.type, { kind: OpenTelemetryApi.SpanKind.INTERNAL }, ctx)
+
+    await handler.handle(domainEvent)
+
+    this.internalSpan.end()
+
+    this.currentSpan.end()
+
+    this.internalSpan = undefined
+    this.currentSpan = undefined
+  }
+
+  async handleError(error: Error): Promise<void> {
+    if (this.internalSpan) {
+      this.internalSpan.recordException(error)
+      this.internalSpan.end()
+      this.internalSpan = undefined
+    }
+
+    if (this.currentSpan) {
+      this.currentSpan.end()
+      this.currentSpan = undefined
+    }
+
+    this.logger.error('Error occured while handling SQS message: %O', error)
+  }
+}

packages/domain-events-infra/src/Infra/index.ts

@@ -1,6 +1,9 @@
 export * from './DirectCall/DirectCallDomainEventPublisher'
 export * from './DirectCall/DirectCallEventMessageHandler'
 
+export * from './OpenTelemetry/OpenTelemetrySDK'
+export * from './OpenTelemetry/OpenTelemetrySDKInterface'
+
 export * from './Redis/RedisDomainEventPublisher'
 export * from './Redis/RedisDomainEventSubscriber'
 export * from './Redis/RedisDomainEventSubscriberFactory'
@@ -12,3 +15,4 @@ export * from './SQS/SQSNewRelicBounceNotificiationHandler'
 export * from './SQS/SQSDomainEventSubscriberFactory'
 export * from './SQS/SQSEventMessageHandler'
 export * from './SQS/SQSNewRelicEventMessageHandler'
+export * from './SQS/SQSOpenTelemetryEventMessageHandler'

packages/event-store/CHANGELOG.md

@@ -3,6 +3,30 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.12.18](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.17...@standardnotes/event-store@1.12.18) (2023-10-10)
+
+**Note:** Version bump only for package @standardnotes/event-store
+
+## [1.12.17](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.16...@standardnotes/event-store@1.12.17) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/event-store
+
+## [1.12.16](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.15...@standardnotes/event-store@1.12.16) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/event-store
+
+## [1.12.15](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.14...@standardnotes/event-store@1.12.15) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/event-store
+
+## [1.12.14](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.13...@standardnotes/event-store@1.12.14) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/event-store
+
+## [1.12.13](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.12...@standardnotes/event-store@1.12.13) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/event-store
+
 ## [1.12.12](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.11...@standardnotes/event-store@1.12.12) (2023-10-09)
 
 **Note:** Version bump only for package @standardnotes/event-store

packages/event-store/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/event-store",
-  "version": "1.12.12",
+  "version": "1.12.18",
   "description": "Event Store Service",
   "private": true,
   "main": "dist/src/index.js",

packages/files/CHANGELOG.md

@@ -3,6 +3,40 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.28.4](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.28.3...@standardnotes/files-server@1.28.4) (2023-10-10)
+
+**Note:** Version bump only for package @standardnotes/files-server
+
+## [1.28.3](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.28.2...@standardnotes/files-server@1.28.3) (2023-10-09)
+
+### Reverts
+
+* Revert "Revert "fix: setting parent span on workers"" ([76ae6f5](https://github.com/standardnotes/files/commit/76ae6f5a882a82ab5f635452e3bc7b2b16709531))
+
+## [1.28.2](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.28.1...@standardnotes/files-server@1.28.2) (2023-10-09)
+
+### Reverts
+
+* Revert "fix: setting parent span on workers" ([3fc07a5](https://github.com/standardnotes/files/commit/3fc07a5b60c26b583efd88e8a80d4c4321e71efb))
+
+## [1.28.1](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.28.0...@standardnotes/files-server@1.28.1) (2023-10-09)
+
+### Bug Fixes
+
+* setting parent span on workers ([1c54d18](https://github.com/standardnotes/files/commit/1c54d18c3ca75353701ba921492a5ecfaa2e3572))
+
+# [1.28.0](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.27.0...@standardnotes/files-server@1.28.0) (2023-10-09)
+
+### Features
+
+* add custom tracing on workers ([65ced2c](https://github.com/standardnotes/files/commit/65ced2cc7b0686dc8af5cdad499412fc8fd29d1d))
+
+# [1.27.0](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.26.4...@standardnotes/files-server@1.27.0) (2023-10-09)
+
+### Features
+
+* add opentelemetry to all services ([5e930d0](https://github.com/standardnotes/files/commit/5e930d08eb60a0da800081342315e7edaf130951))
+
 ## [1.26.4](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.26.3...@standardnotes/files-server@1.26.4) (2023-10-09)
 
 ### Bug Fixes

packages/files/bin/server.ts

@@ -17,6 +17,7 @@ import { InversifyExpressServer } from 'inversify-express-utils'
 import { ContainerConfigLoader } from '../src/Bootstrap/Container'
 import TYPES from '../src/Bootstrap/Types'
 import { Env } from '../src/Bootstrap/Env'
+import { OpenTelemetrySDKInterface } from '@standardnotes/domain-events-infra'
 
 const container = new ContainerConfigLoader()
 void container.load().then((container) => {
@@ -90,6 +91,11 @@ void container.load().then((container) => {
 
   const serverInstance = server.build()
 
+  if (!container.get<boolean>(TYPES.Files_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)) {
+    const openTelemetrySDK = container.get<OpenTelemetrySDKInterface>(TYPES.Files_OpenTelemetrySDK)
+    openTelemetrySDK.start()
+  }
+
   serverInstance.listen(env.get('PORT'))
 
   logger.info(`Server started on port ${process.env.PORT}`)

packages/files/bin/worker.ts

@@ -8,8 +8,9 @@ import { Env } from '../src/Bootstrap/Env'
 import { DomainEventSubscriberFactoryInterface } from '@standardnotes/domain-events'
 import * as dayjs from 'dayjs'
 import * as utc from 'dayjs/plugin/utc'
+import { OpenTelemetrySDKInterface } from '@standardnotes/domain-events-infra'
 
-const container = new ContainerConfigLoader()
+const container = new ContainerConfigLoader('worker')
 void container.load().then((container) => {
   dayjs.extend(utc)
 
@@ -20,6 +21,11 @@ void container.load().then((container) => {
 
   logger.info('Starting worker...')
 
+  if (!container.get<boolean>(TYPES.Files_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)) {
+    const openTelemetrySDK = container.get<OpenTelemetrySDKInterface>(TYPES.Files_OpenTelemetrySDK)
+    openTelemetrySDK.start()
+  }
+
   const subscriberFactory: DomainEventSubscriberFactoryInterface = container.get(
     TYPES.Files_DomainEventSubscriberFactory,
   )

packages/files/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/files-server",
-  "version": "1.26.4",
+  "version": "1.28.4",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/files/src/Bootstrap/Container.ts

@@ -16,9 +16,12 @@ import { DomainEventFactory } from '../Domain/Event/DomainEventFactory'
 import {
   DirectCallDomainEventPublisher,
   DirectCallEventMessageHandler,
+  OpenTelemetrySDK,
+  OpenTelemetrySDKInterface,
   SNSDomainEventPublisher,
   SQSDomainEventSubscriberFactory,
   SQSEventMessageHandler,
+  SQSOpenTelemetryEventMessageHandler,
 } from '@standardnotes/domain-events-infra'
 import { StreamDownloadFile } from '../Domain/UseCase/StreamDownloadFile/StreamDownloadFile'
 import { FileDownloaderInterface } from '../Domain/Services/FileDownloaderInterface'
@@ -52,8 +55,11 @@ import { S3FileMover } from '../Infra/S3/S3FileMover'
 import { FSFileMover } from '../Infra/FS/FSFileMover'
 import { MoveFile } from '../Domain/UseCase/MoveFile/MoveFile'
 import { SharedVaultValetTokenAuthMiddleware } from '../Infra/InversifyExpress/Middleware/SharedVaultValetTokenAuthMiddleware'
+import { ServiceIdentifier } from '@standardnotes/domain-core'
 
 export class ContainerConfigLoader {
+  constructor(private mode: 'server' | 'worker' = 'server') {}
+
   async load(configuration?: {
     directCallDomainEventPublisher?: DirectCallDomainEventPublisher
     logger?: Transform
@@ -81,8 +87,24 @@ export class ContainerConfigLoader {
       .bind(TYPES.Files_FILE_UPLOAD_PATH)
       .toConstantValue(env.get('FILE_UPLOAD_PATH', true) ?? `${__dirname}/../../uploads`)
 
-    const isConfiguredForHomeServer = env.get('MODE', true) === 'home-server'
     const isConfiguredForInMemoryCache = env.get('CACHE_TYPE', true) === 'memory'
+    const isConfiguredForHomeServer = env.get('MODE', true) === 'home-server'
+    const isConfiguredForSelfHosting = env.get('MODE', true) === 'self-hosted'
+    const isConfiguredForHomeServerOrSelfHosting = isConfiguredForHomeServer || isConfiguredForSelfHosting
+
+    container
+      .bind<boolean>(TYPES.Files_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)
+      .toConstantValue(isConfiguredForHomeServerOrSelfHosting)
+
+    if (!isConfiguredForHomeServerOrSelfHosting) {
+      container
+        .bind<OpenTelemetrySDKInterface>(TYPES.Files_OpenTelemetrySDK)
+        .toConstantValue(
+          new OpenTelemetrySDK(
+            this.mode === 'server' ? ServiceIdentifier.NAMES.Files : ServiceIdentifier.NAMES.FilesWorker,
+          ),
+        )
+    }
 
     let logger: winston.Logger
     if (configuration?.logger) {
@@ -291,7 +313,15 @@ export class ContainerConfigLoader {
     } else {
       container
         .bind<DomainEventMessageHandlerInterface>(TYPES.Files_DomainEventMessageHandler)
-        .toConstantValue(new SQSEventMessageHandler(eventHandlers, container.get(TYPES.Files_Logger)))
+        .toConstantValue(
+          isConfiguredForHomeServerOrSelfHosting
+            ? new SQSEventMessageHandler(eventHandlers, container.get(TYPES.Files_Logger))
+            : new SQSOpenTelemetryEventMessageHandler(
+                ServiceIdentifier.NAMES.FilesWorker,
+                eventHandlers,
+                container.get(TYPES.Files_Logger),
+              ),
+        )
       container
         .bind<DomainEventSubscriberFactoryInterface>(TYPES.Files_DomainEventSubscriberFactory)
         .toConstantValue(

packages/files/src/Bootstrap/Types.ts

@@ -5,6 +5,7 @@ const TYPES = {
   Files_S3: Symbol.for('Files_S3'),
   Files_SNS: Symbol.for('Files_SNS'),
   Files_SQS: Symbol.for('Files_SQS'),
+  Files_OpenTelemetrySDK: Symbol.for('Files_OpenTelemetrySDK'),
 
   // use cases
   Files_UploadFileChunk: Symbol.for('Files_UploadFileChunk'),
@@ -47,6 +48,9 @@ const TYPES = {
   Files_VERSION: Symbol.for('Files_VERSION'),
   Files_NEW_RELIC_ENABLED: Symbol.for('Files_NEW_RELIC_ENABLED'),
   Files_FILE_UPLOAD_PATH: Symbol.for('Files_FILE_UPLOAD_PATH'),
+  Files_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING: Symbol.for(
+    'Files_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING',
+  ),
 
   // Handlers
   Files_DomainEventMessageHandler: Symbol.for('Files_DomainEventMessageHandler'),

packages/home-server/CHANGELOG.md

@@ -3,6 +3,42 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.16.45](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.44...@standardnotes/home-server@1.16.45) (2023-10-10)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.16.44](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.43...@standardnotes/home-server@1.16.44) (2023-10-10)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.16.43](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.42...@standardnotes/home-server@1.16.43) (2023-10-10)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.16.42](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.41...@standardnotes/home-server@1.16.42) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.16.41](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.40...@standardnotes/home-server@1.16.41) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.16.40](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.39...@standardnotes/home-server@1.16.40) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.16.39](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.38...@standardnotes/home-server@1.16.39) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.16.38](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.37...@standardnotes/home-server@1.16.38) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.16.37](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.36...@standardnotes/home-server@1.16.37) (2023-10-09)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
 ## [1.16.36](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.35...@standardnotes/home-server@1.16.36) (2023-10-09)
 
 **Note:** Version bump only for package @standardnotes/home-server

packages/home-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/home-server",
-  "version": "1.16.36",
+  "version": "1.16.45",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/revisions/CHANGELOG.md

@@ -3,6 +3,66 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.43.2](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.43.1...@standardnotes/revisions-server@1.43.2) (2023-10-10)
+
+### Bug Fixes
+
+* **revisions:** check for duplicates during transition ([4629580](https://github.com/standardnotes/server/commit/4629580650f2b70e736ed4464b311241c563074c))
+
+## [1.43.1](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.43.0...@standardnotes/revisions-server@1.43.1) (2023-10-10)
+
+### Bug Fixes
+
+* transition logs to be more verbose ([036317e](https://github.com/standardnotes/server/commit/036317e33347f121fa799cbd409f85759798369d))
+
+# [1.43.0](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.42.3...@standardnotes/revisions-server@1.43.0) (2023-10-10)
+
+### Features
+
+* add more logs to transition ([783fd9e](https://github.com/standardnotes/server/commit/783fd9e2c6350a7f0f1e8d009d01a3328564aca0))
+
+## [1.42.3](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.42.2...@standardnotes/revisions-server@1.42.3) (2023-10-09)
+
+### Reverts
+
+* Revert "Revert "fix: setting parent span on workers"" ([76ae6f5](https://github.com/standardnotes/server/commit/76ae6f5a882a82ab5f635452e3bc7b2b16709531))
+
+## [1.42.2](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.42.1...@standardnotes/revisions-server@1.42.2) (2023-10-09)
+
+### Reverts
+
+* Revert "fix: setting parent span on workers" ([3fc07a5](https://github.com/standardnotes/server/commit/3fc07a5b60c26b583efd88e8a80d4c4321e71efb))
+
+## [1.42.1](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.42.0...@standardnotes/revisions-server@1.42.1) (2023-10-09)
+
+### Bug Fixes
+
+* setting parent span on workers ([1c54d18](https://github.com/standardnotes/server/commit/1c54d18c3ca75353701ba921492a5ecfaa2e3572))
+
+# [1.42.0](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.41.0...@standardnotes/revisions-server@1.42.0) (2023-10-09)
+
+### Features
+
+* add custom tracing on workers ([65ced2c](https://github.com/standardnotes/server/commit/65ced2cc7b0686dc8af5cdad499412fc8fd29d1d))
+
+# [1.41.0](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.40.4...@standardnotes/revisions-server@1.41.0) (2023-10-09)
+
+### Bug Fixes
+
+* **revisions:** add trace id sampler ([bf9072d](https://github.com/standardnotes/server/commit/bf9072dbd473ef68df285d5729ffcb2383f5fa3d))
+* **revisions:** instrumenting opentelemetry on non production ([0c89d8b](https://github.com/standardnotes/server/commit/0c89d8bd2d90fd9875d331baa888706b4f4ca4a6))
+
+### Features
+
+* add opentelemetry to all services ([5e930d0](https://github.com/standardnotes/server/commit/5e930d08eb60a0da800081342315e7edaf130951))
+* **revisions:** switch to open telemetry ([6abdd73](https://github.com/standardnotes/server/commit/6abdd73a3123c35e2df492711f785f9a30571b22))
+
+## [1.40.4](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.40.3...@standardnotes/revisions-server@1.40.4) (2023-10-09)
+
+### Bug Fixes
+
+* **revisions:** creation date formatting ([4655308](https://github.com/standardnotes/server/commit/465530841f49df1bda44d9a9552306279575f57b))
+
 ## [1.40.3](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.40.2...@standardnotes/revisions-server@1.40.3) (2023-10-09)
 
 ### Bug Fixes

packages/revisions/bin/server.ts

@@ -11,6 +11,7 @@ import { ContainerConfigLoader } from '../src/Bootstrap/Container'
 
 import '../src/Infra/InversifyExpress/AnnotatedRevisionsController'
 import '../src/Infra/InversifyExpress/AnnotatedHealthCheckController'
+import { OpenTelemetrySDKInterface } from '@standardnotes/domain-events-infra'
 
 const container = new ContainerConfigLoader()
 void container.load().then((container) => {
@@ -45,6 +46,11 @@ void container.load().then((container) => {
 
   const serverInstance = server.build()
 
+  if (!container.get<boolean>(TYPES.Revisions_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)) {
+    const openTelemetrySDK = container.get<OpenTelemetrySDKInterface>(TYPES.Revisions_OpenTelemetrySDK)
+    openTelemetrySDK.start()
+  }
+
   serverInstance.listen(env.get('PORT'))
 
   logger.info(`Server started on port ${process.env.PORT}`)

packages/revisions/bin/worker.ts

@@ -6,6 +6,7 @@ import TYPES from '../src/Bootstrap/Types'
 import { Env } from '../src/Bootstrap/Env'
 import { DomainEventSubscriberFactoryInterface } from '@standardnotes/domain-events'
 import { ContainerConfigLoader } from '../src/Bootstrap/Container'
+import { OpenTelemetrySDKInterface } from '@standardnotes/domain-events-infra'
 
 const container = new ContainerConfigLoader('worker')
 void container.load().then((container) => {
@@ -16,6 +17,11 @@ void container.load().then((container) => {
 
   logger.info('Starting worker...')
 
+  if (!container.get<boolean>(TYPES.Revisions_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)) {
+    const openTelemetrySDK = container.get<OpenTelemetrySDKInterface>(TYPES.Revisions_OpenTelemetrySDK)
+    openTelemetrySDK.start()
+  }
+
   const subscriberFactory: DomainEventSubscriberFactoryInterface = container.get(
     TYPES.Revisions_DomainEventSubscriberFactory,
   )

packages/revisions/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/revisions-server",
-  "version": "1.40.3",
+  "version": "1.43.2",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/revisions/src/Bootstrap/Container.ts

@@ -1,4 +1,9 @@
-import { ControllerContainer, ControllerContainerInterface, MapperInterface } from '@standardnotes/domain-core'
+import {
+  ControllerContainer,
+  ControllerContainerInterface,
+  MapperInterface,
+  ServiceIdentifier,
+} from '@standardnotes/domain-core'
 import Redis from 'ioredis'
 import { Container, interfaces } from 'inversify'
 import { MongoRepository, Repository } from 'typeorm'
@@ -35,6 +40,9 @@ import {
   DirectCallEventMessageHandler,
   DirectCallDomainEventPublisher,
   SNSDomainEventPublisher,
+  OpenTelemetrySDKInterface,
+  OpenTelemetrySDK,
+  SQSOpenTelemetryEventMessageHandler,
 } from '@standardnotes/domain-events-infra'
 import { DumpRepositoryInterface } from '../Domain/Dump/DumpRepositoryInterface'
 import { AccountDeletionRequestedEventHandler } from '../Domain/Handler/AccountDeletionRequestedEventHandler'
@@ -97,6 +105,10 @@ export class ContainerConfigLoader {
       defaultScope: 'Singleton',
     })
 
+    container
+      .bind<boolean>(TYPES.Revisions_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)
+      .toConstantValue(isConfiguredForHomeServerOrSelfHosting)
+
     if (!isConfiguredForInMemoryCache) {
       const redisUrl = env.get('REDIS_URL')
       const isRedisInClusterMode = redisUrl.indexOf(',') > 0
@@ -147,6 +159,16 @@ export class ContainerConfigLoader {
     container.bind(TYPES.Revisions_NEW_RELIC_ENABLED).toConstantValue(env.get('NEW_RELIC_ENABLED', true))
     container.bind(TYPES.Revisions_VERSION).toConstantValue(env.get('VERSION', true) ?? 'development')
 
+    if (!isConfiguredForHomeServerOrSelfHosting) {
+      container
+        .bind<OpenTelemetrySDKInterface>(TYPES.Revisions_OpenTelemetrySDK)
+        .toConstantValue(
+          new OpenTelemetrySDK(
+            this.mode === 'server' ? ServiceIdentifier.NAMES.Revisions : ServiceIdentifier.NAMES.RevisionsWorker,
+          ),
+        )
+    }
+
     if (!isConfiguredForHomeServer) {
       // env vars
       container.bind(TYPES.Revisions_SNS_TOPIC_ARN).toConstantValue(env.get('SNS_TOPIC_ARN'))
@@ -237,10 +259,10 @@ export class ContainerConfigLoader {
       .toConstantValue(new SQLRevisionMetadataPersistenceMapper())
     container
       .bind<MapperInterface<Revision, SQLLegacyRevision>>(TYPES.Revisions_SQLLegacyRevisionPersistenceMapper)
-      .toConstantValue(new SQLLegacyRevisionPersistenceMapper())
+      .toConstantValue(new SQLLegacyRevisionPersistenceMapper(container.get<TimerInterface>(TYPES.Revisions_Timer)))
     container
       .bind<MapperInterface<Revision, SQLRevision>>(TYPES.Revisions_SQLRevisionPersistenceMapper)
-      .toConstantValue(new SQLRevisionPersistenceMapper())
+      .toConstantValue(new SQLRevisionPersistenceMapper(container.get<TimerInterface>(TYPES.Revisions_Timer)))
     container
       .bind<MapperInterface<RevisionMetadata, MongoDBRevision>>(
         TYPES.Revisions_MongoDBRevisionMetadataPersistenceMapper,
@@ -508,7 +530,15 @@ export class ContainerConfigLoader {
     } else {
       container
         .bind<DomainEventMessageHandlerInterface>(TYPES.Revisions_DomainEventMessageHandler)
-        .toConstantValue(new SQSEventMessageHandler(eventHandlers, container.get(TYPES.Revisions_Logger)))
+        .toConstantValue(
+          isConfiguredForHomeServerOrSelfHosting
+            ? new SQSEventMessageHandler(eventHandlers, container.get(TYPES.Revisions_Logger))
+            : new SQSOpenTelemetryEventMessageHandler(
+                ServiceIdentifier.NAMES.RevisionsWorker,
+                eventHandlers,
+                container.get(TYPES.Revisions_Logger),
+              ),
+        )
 
       container
         .bind<DomainEventSubscriberFactoryInterface>(TYPES.Revisions_DomainEventSubscriberFactory)

packages/revisions/src/Bootstrap/Types.ts

@@ -5,6 +5,7 @@ const TYPES = {
   Revisions_SQS: Symbol.for('Revisions_SQS'),
   Revisions_SNS: Symbol.for('Revisions_SNS'),
   Revisions_S3: Symbol.for('Revisions_S3'),
+  Revisions_OpenTelemetrySDK: Symbol.for('Revisions_OpenTelemetrySDK'),
   Revisions_Env: Symbol.for('Revisions_Env'),
   // Map
   Revisions_SQLLegacyRevisionMetadataPersistenceMapper: Symbol.for(
@@ -39,6 +40,9 @@ const TYPES = {
   Revisions_SNS_AWS_REGION: Symbol.for('Revisions_SNS_AWS_REGION'),
   Revisions_NEW_RELIC_ENABLED: Symbol.for('Revisions_NEW_RELIC_ENABLED'),
   Revisions_VERSION: Symbol.for('Revisions_VERSION'),
+  Revisions_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING: Symbol.for(
+    'Revisions_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING',
+  ),
   // use cases
   Revisions_GetRevisionsMetada: Symbol.for('Revisions_GetRevisionsMetada'),
   Revisions_GetRevision: Symbol.for('Revisions_GetRevision'),

packages/revisions/src/Domain/Revision/RevisionRepositoryInterface.ts

@@ -13,5 +13,6 @@ export interface RevisionRepositoryInterface {
   updateUserUuid(itemUuid: Uuid, userUuid: Uuid): Promise<void>
   findByUserUuid(dto: { userUuid: Uuid; offset?: number; limit?: number }): Promise<Array<Revision>>
   insert(revision: Revision): Promise<boolean>
+  update(revision: Revision): Promise<boolean>
   clearSharedVaultAndKeySystemAssociations(dto: { itemUuid?: Uuid; sharedVaultUuid: Uuid }): Promise<void>
 }

packages/revisions/src/Domain/UseCase/Transition/TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser/TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser.ts

@@ -112,7 +112,15 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
       const totalRevisionsCountForUser = await (
         this.secondRevisionsRepository as RevisionRepositoryInterface
       ).countByUserUuid(userUuid)
+      this.logger.info(`[${userUuid.value}] Total revisions count for user: ${totalRevisionsCountForUser}`)
       const totalPages = Math.ceil(totalRevisionsCountForUser / this.pageSize)
+      this.logger.info(`[${userUuid.value}] Total pages: ${totalPages}`)
+      let insertedCount = 0
+      let newerCount = 0
+      let identicalCount = 0
+      let updatedCount = 0
+      let duplicatedCount = 0
+      const processedUuids = new Set<string>()
       for (let currentPage = initialPage; currentPage <= totalPages; currentPage++) {
         const isPageInEvery10Percent = currentPage % Math.ceil(totalPages / 10) === 0
         if (isPageInEvery10Percent) {
@@ -121,6 +129,9 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
               (currentPage / totalPages) * 100,
             )}% completed`,
           )
+          this.logger.info(
+            `[${userUuid.value}] Inserted ${insertedCount} revisions so far. Skipped ${newerCount} revisions because they were newer in primary database. Skipped ${identicalCount} revisions because they were identical in primary and secondary database. Updated ${updatedCount} revisions because they were older in primary database.`,
+          )
           await this.updateTransitionStatus(userUuid, TransitionStatus.STATUSES.InProgress, timestamp)
         }
 
@@ -138,42 +149,44 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
         const revisions = await (this.secondRevisionsRepository as RevisionRepositoryInterface).findByUserUuid(query)
         for (const revision of revisions) {
           try {
+            if (processedUuids.has(revision.id.toString())) {
+              this.logger.warn(`[${userUuid.value}] Revision ${revision.id.toString()} was already processed`)
+              duplicatedCount++
+            } else {
+              processedUuids.add(revision.id.toString())
+            }
+
             const revisionInPrimary = await this.primaryRevisionsRepository.findOneByUuid(
               Uuid.create(revision.id.toString()).getValue(),
               revision.props.userUuid as Uuid,
               [],
             )
 
-            if (revisionInPrimary !== null) {
+            if (!revisionInPrimary) {
+              await this.primaryRevisionsRepository.insert(revision)
+
+              insertedCount++
+            } else {
               if (revisionInPrimary.props.dates.updatedAt > revision.props.dates.updatedAt) {
                 this.logger.info(
                   `[${
                     userUuid.value
                   }] Revision ${revision.id.toString()} is older in secondary than revision in primary database`,
                 )
+                newerCount++
 
                 continue
               }
+
               if (revisionInPrimary.isIdenticalTo(revision)) {
+                identicalCount++
+
                 continue
               }
 
-              this.logger.info(
-                `[${userUuid.value}] Removing revision ${revision.id.toString()} in primary database: ${JSON.stringify(
-                  revisionInPrimary,
-                )} as it is not identical to revision in secondary database: ${JSON.stringify(revision)}`,
-              )
+              await this.primaryRevisionsRepository.update(revision)
 
-              await this.primaryRevisionsRepository.removeOneByUuid(
-                Uuid.create(revisionInPrimary.id.toString()).getValue(),
-                revisionInPrimary.props.userUuid as Uuid,
-              )
-              await this.allowForPrimaryDatabaseToCatchUp()
-            }
-
-            const didSave = await this.primaryRevisionsRepository.insert(revision)
-            if (!didSave) {
-              this.logger.error(`Failed to save revision ${revision.id.toString()} to primary database`)
+              updatedCount++
             }
           } catch (error) {
             this.logger.error(
@@ -183,6 +196,13 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
         }
       }
 
+      this.logger.info(
+        `[${userUuid.value}] Inserted ${insertedCount} revisions. Skipped ${newerCount} revisions because they were newer in primary database. Skipped ${identicalCount} revisions because they were identical in primary and secondary database. Updated ${updatedCount} revisions because they were older in primary database.`,
+      )
+      if (duplicatedCount > 0) {
+        this.logger.warn(`[${userUuid.value}] Skipped ${duplicatedCount} duplicated revisions`)
+      }
+
       return Result.ok()
     } catch (error) {
       return Result.fail(`Errored when migrating revisions for user ${userUuid.value}: ${(error as Error).message}`)

packages/revisions/src/Infra/TypeORM/MongoDB/MongoDBRevisionRepository.ts

@@ -193,4 +193,14 @@ export class MongoDBRevisionRepository implements RevisionRepositoryInterface {
 
     return insertResult.acknowledged
   }
+
+  async update(revision: Revision): Promise<boolean> {
+    const persistence = this.revisionMapper.toProjection(revision)
+
+    const { _id, ...rest } = persistence
+
+    const updateResult = await this.mongoRepository.updateOne({ _id: _id }, { $set: rest })
+
+    return updateResult.acknowledged
+  }
 }

packages/revisions/src/Infra/TypeORM/SQL/SQLLegacyRevisionRepository.ts

@@ -111,9 +111,19 @@ export class SQLLegacyRevisionRepository implements RevisionRepositoryInterface
   }
 
   async insert(revision: Revision): Promise<boolean> {
-    const SQLLegacyRevision = this.revisionMapper.toProjection(revision)
+    const projection = this.revisionMapper.toProjection(revision)
 
-    await this.ormRepository.insert(SQLLegacyRevision)
+    await this.ormRepository.insert(projection)
+
+    return true
+  }
+
+  async update(revision: Revision): Promise<boolean> {
+    const projection = this.revisionMapper.toProjection(revision)
+
+    const { uuid, ...rest } = projection
+
+    await this.ormRepository.update({ uuid: uuid }, rest)
 
     return true
   }

packages/revisions/src/Mapping/Persistence/SQL/SQLLegacyRevisionPersistenceMapper.ts

@@ -1,9 +1,12 @@
 import { MapperInterface, Dates, UniqueEntityId, Uuid, ContentType } from '@standardnotes/domain-core'
+import { TimerInterface } from '@standardnotes/time'
 
 import { Revision } from '../../../Domain/Revision/Revision'
 import { SQLLegacyRevision } from '../../../Infra/TypeORM/SQL/SQLLegacyRevision'
 
 export class SQLLegacyRevisionPersistenceMapper implements MapperInterface<Revision, SQLLegacyRevision> {
+  constructor(private timer: TimerInterface) {}
+
   toDomain(projection: SQLLegacyRevision): Revision {
     const contentTypeOrError = ContentType.create(projection.contentType)
     if (contentTypeOrError.isFailed()) {
@@ -37,7 +40,7 @@ export class SQLLegacyRevisionPersistenceMapper implements MapperInterface<Revis
         authHash: projection.authHash,
         content: projection.content,
         contentType,
-        creationDate: projection.creationDate,
+        creationDate: new Date(this.timer.convertDateToFormattedString(projection.creationDate, 'YYYY-MM-DD')),
         encItemKey: projection.encItemKey,
         itemsKeyId: projection.itemsKeyId,
         itemUuid,
@@ -61,7 +64,9 @@ export class SQLLegacyRevisionPersistenceMapper implements MapperInterface<Revis
     sqlRevision.contentType = domain.props.contentType.value
     sqlRevision.createdAt = domain.props.dates.createdAt
     sqlRevision.updatedAt = domain.props.dates.updatedAt
-    sqlRevision.creationDate = domain.props.creationDate
+    sqlRevision.creationDate = new Date(
+      this.timer.convertDateToFormattedString(domain.props.creationDate, 'YYYY-MM-DD'),
+    )
     sqlRevision.encItemKey = domain.props.encItemKey
     sqlRevision.itemUuid = domain.props.itemUuid.value
     sqlRevision.itemsKeyId = domain.props.itemsKeyId

packages/revisions/src/Mapping/Persistence/SQL/SQLRevisionPersistenceMapper.ts

@@ -1,4 +1,5 @@
 import { MapperInterface, Dates, UniqueEntityId, Uuid, ContentType } from '@standardnotes/domain-core'
+import { TimerInterface } from '@standardnotes/time'
 
 import { Revision } from '../../../Domain/Revision/Revision'
 import { SQLRevision } from '../../../Infra/TypeORM/SQL/SQLRevision'
@@ -6,6 +7,8 @@ import { SharedVaultAssociation } from '../../../Domain/SharedVault/SharedVaultA
 import { KeySystemAssociation } from '../../../Domain/KeySystem/KeySystemAssociation'
 
 export class SQLRevisionPersistenceMapper implements MapperInterface<Revision, SQLRevision> {
+  constructor(private timer: TimerInterface) {}
+
   toDomain(projection: SQLRevision): Revision {
     const contentTypeOrError = ContentType.create(projection.contentType)
     if (contentTypeOrError.isFailed()) {
@@ -72,7 +75,7 @@ export class SQLRevisionPersistenceMapper implements MapperInterface<Revision, S
         authHash: projection.authHash,
         content: projection.content,
         contentType,
-        creationDate: projection.creationDate,
+        creationDate: new Date(this.timer.convertDateToFormattedString(projection.creationDate, 'YYYY-MM-DD')),
         encItemKey: projection.encItemKey,
         itemsKeyId: projection.itemsKeyId,
         itemUuid,
@@ -98,7 +101,9 @@ export class SQLRevisionPersistenceMapper implements MapperInterface<Revision, S
     sqlRevision.contentType = domain.props.contentType.value
     sqlRevision.createdAt = domain.props.dates.createdAt
     sqlRevision.updatedAt = domain.props.dates.updatedAt
-    sqlRevision.creationDate = domain.props.creationDate
+    sqlRevision.creationDate = new Date(
+      this.timer.convertDateToFormattedString(domain.props.creationDate, 'YYYY-MM-DD'),
+    )
     sqlRevision.encItemKey = domain.props.encItemKey
     sqlRevision.itemUuid = domain.props.itemUuid.value
     sqlRevision.itemsKeyId = domain.props.itemsKeyId