chore(release): publish new version

- @standardnotes/analytics@2.26.17
 - @standardnotes/api-gateway@1.74.15
 - @standardnotes/auth-server@1.143.7
 - @standardnotes/domain-core@1.30.1
 - @standardnotes/event-store@1.11.45
 - @standardnotes/files-server@1.22.24
 - @standardnotes/home-server@1.15.68
 - @standardnotes/revisions-server@1.35.7
 - @standardnotes/scheduler-server@1.20.49
 - @standardnotes/settings@1.21.34
 - @standardnotes/syncing-server@1.98.6
 - @standardnotes/websockets-server@1.10.46

.github/workflows/common-docker-image.yml

@@ -33,17 +33,12 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Create Bundle Dir
-      id: bundle-dir
-      run: echo "temp_dir=$(mktemp -d -t ${{ inputs.service_name }}-${{ github.sha }}-XXXXXXX)" >> $GITHUB_OUTPUT
-
     - name: Cache build
       id: cache-build
       uses: actions/cache@v3
       with:
         path: |
           packages/**/dist
-          ${{ steps.bundle-dir.outputs.temp_dir }}
         key: ${{ runner.os }}-${{ inputs.service_name }}-build-${{ github.sha }}
 
     - name: Set up Node
@@ -57,11 +52,7 @@ jobs:
 
     - name: Build
       if: steps.cache-build.outputs.cache-hit != 'true'
-      run: yarn build ${{ inputs.package_path }}
+      run: yarn build
-
-    - name: Bundle
-      if: steps.cache-build.outputs.cache-hit != 'true'
-      run: yarn workspace ${{ inputs.workspace_name }} bundle --no-compress --output-directory ${{ steps.bundle-dir.outputs.temp_dir }}
 
     - name: Login to Docker Hub
       uses: docker/login-action@v2
@@ -93,8 +84,8 @@ jobs:
       uses: docker/build-push-action@v4
       with:
         builder: ${{ steps.buildx.outputs.name }}
-        context: ${{ steps.bundle-dir.outputs.temp_dir }}
+        context: .
-        file: ${{ steps.bundle-dir.outputs.temp_dir }}/${{ inputs.package_path }}/Dockerfile
+        file: ${{ inputs.package_path }}/Dockerfile
         platforms: linux/amd64,linux/arm64
         push: true
         tags: |

package.json

@@ -20,8 +20,8 @@
     "release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"",
     "publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose",
     "postversion": "./scripts/push-tags-one-by-one.sh",
-    "e2e": "yarn build packages/home-server && PORT=3123 yarn workspace @standardnotes/home-server start",
+    "e2e": "yarn workspace @standardnotes/home-server run build && PORT=3123 yarn workspace @standardnotes/home-server start",
-    "start": "yarn build packages/home-server && yarn workspace @standardnotes/home-server start"
+    "start": "yarn workspace @standardnotes/home-server run build && yarn workspace @standardnotes/home-server start"
   },
   "devDependencies": {
     "@commitlint/cli": "^17.0.2",

packages/analytics/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [2.26.17](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.26.16...@standardnotes/analytics@2.26.17) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/analytics
+
 ## [2.26.16](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.26.15...@standardnotes/analytics@2.26.16) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/analytics

packages/analytics/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/analytics",
-  "version": "2.26.16",
+  "version": "2.26.17",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/api-gateway/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.74.15](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.74.14...@standardnotes/api-gateway@1.74.15) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/api-gateway
+
 ## [1.74.14](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.74.13...@standardnotes/api-gateway@1.74.14) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/api-gateway

packages/api-gateway/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/api-gateway",
-  "version": "1.74.14",
+  "version": "1.74.15",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/auth/CHANGELOG.md

@@ -3,6 +3,22 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.143.7](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.143.6...@standardnotes/auth-server@1.143.7) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/auth-server
+
+## [1.143.6](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.143.4...@standardnotes/auth-server@1.143.6) (2023-09-19)
+
+### Bug Fixes
+
+* **auth:** bump version ([b91cd7e](https://github.com/standardnotes/server/commit/b91cd7e2329d68026665f61d9bd17ba918c57563))
+
+## [1.143.4](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.143.3...@standardnotes/auth-server@1.143.4) (2023-09-19)
+
+### Bug Fixes
+
+* **auth:** add debug logs to transition ([bf855bb](https://github.com/standardnotes/server/commit/bf855bb26e8a3618113bd2f5801c260782566815))
+
 ## [1.143.3](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.143.2...@standardnotes/auth-server@1.143.3) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/auth-server

packages/auth/bin/transition.ts

@@ -53,7 +53,10 @@ const requestTransition = async (
       continue
     }
 
+    let wasTransitionRequested = false
+
     if (itemsTransitionStatus?.value !== TransitionStatus.STATUSES.Verified) {
+      wasTransitionRequested = true
       await transitionStatusRepository.remove(user.uuid, 'items')
 
       await domainEventPublisher.publish(
@@ -66,6 +69,7 @@ const requestTransition = async (
     }
 
     if (revisionsTransitionStatus?.value !== TransitionStatus.STATUSES.Verified) {
+      wasTransitionRequested = true
       await transitionStatusRepository.remove(user.uuid, 'revisions')
 
       await domainEventPublisher.publish(
@@ -78,6 +82,12 @@ const requestTransition = async (
     }
 
     usersTriggered += 1
+
+    if (wasTransitionRequested) {
+      logger.info(
+        `[TRANSITION ${timestamp}] Transition requested for user ${user.uuid} - items status: ${itemsTransitionStatus?.value}, revisions status: ${revisionsTransitionStatus?.value}, has transition role: ${userHasTransitionRole}`,
+      )
+    }
   }
 
   logger.info(

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/auth-server",
-  "version": "1.143.3",
+  "version": "1.143.7",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/domain-core/CHANGELOG.md

@@ -3,6 +3,12 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.30.1](https://github.com/standardnotes/server/compare/@standardnotes/domain-core@1.30.0...@standardnotes/domain-core@1.30.1) (2023-09-19)
+
+### Bug Fixes
+
+* **domain-core:** allow any version and variant of the UUID format ([de081fe](https://github.com/standardnotes/server/commit/de081fe78658bdd36c8c5d86b70a16a2880aa3d1))
+
 # [1.30.0](https://github.com/standardnotes/server/compare/@standardnotes/domain-core@1.29.0...@standardnotes/domain-core@1.30.0) (2023-09-18)
 
 ### Features

packages/domain-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/domain-core",
-  "version": "1.30.0",
+  "version": "1.30.1",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/domain-core/src/Domain/Core/Validator.spec.ts

@@ -5,6 +5,7 @@ describe('Validator', () => {
     '2221101c-1da9-4d2b-9b32-b8be2a8d1c82',
     'c08f2f29-a74b-42b4-aefd-98af9832391c',
     'b453fa64-1493-443b-b5bb-bca7b9c696c7',
+    'fa7350b3-77cf-8c0c-40b2-6046b13254fe',
   ]
 
   const invalidUuids = [

packages/domain-core/src/Domain/Core/Validator.ts

@@ -1,12 +1,13 @@
 import { Result } from './Result'
 
 export class Validator {
-  private static readonly UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i
+  private static readonly UUID_ANY_VERSION_AND_VARIANT_REGEX =
+    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
   private static readonly EMAIL_REGEX =
     /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/
 
   static isValidUuid(value: string): Result<string> {
-    const matchesUuidRegex = String(value).toLowerCase().match(Validator.UUID_REGEX) !== null
+    const matchesUuidRegex = String(value).toLowerCase().match(Validator.UUID_ANY_VERSION_AND_VARIANT_REGEX) !== null
     if (matchesUuidRegex) {
       return Result.ok()
     }

packages/event-store/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.11.45](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.11.44...@standardnotes/event-store@1.11.45) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/event-store
+
 ## [1.11.44](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.11.43...@standardnotes/event-store@1.11.44) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/event-store

packages/event-store/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/event-store",
-  "version": "1.11.44",
+  "version": "1.11.45",
   "description": "Event Store Service",
   "private": true,
   "main": "dist/src/index.js",

packages/files/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.22.24](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.22.23...@standardnotes/files-server@1.22.24) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/files-server
+
 ## [1.22.23](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.22.22...@standardnotes/files-server@1.22.23) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/files-server

packages/files/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/files-server",
-  "version": "1.22.23",
+  "version": "1.22.24",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/home-server/CHANGELOG.md

@@ -3,6 +3,34 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.15.68](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.67...@standardnotes/home-server@1.15.68) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.15.67](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.66...@standardnotes/home-server@1.15.67) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.15.66](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.65...@standardnotes/home-server@1.15.66) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.15.65](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.64...@standardnotes/home-server@1.15.65) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.15.64](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.63...@standardnotes/home-server@1.15.64) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.15.63](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.62...@standardnotes/home-server@1.15.63) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.15.62](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.61...@standardnotes/home-server@1.15.62) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
 ## [1.15.61](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.60...@standardnotes/home-server@1.15.61) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/home-server

packages/home-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/home-server",
-  "version": "1.15.61",
+  "version": "1.15.68",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/revisions/CHANGELOG.md

@@ -3,6 +3,38 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.35.7](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.35.6...@standardnotes/revisions-server@1.35.7) (2023-09-19)
+
+### Bug Fixes
+
+* skip removing already existing content in secondary to pick up where the transition left of ([857c6af](https://github.com/standardnotes/server/commit/857c6af9468ec829ff4dce9a96ba7bf9c14d55a5))
+
+## [1.35.6](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.35.5...@standardnotes/revisions-server@1.35.6) (2023-09-19)
+
+### Bug Fixes
+
+* increase timeout for secondary database to catch up for indexes to be rebuilt ([b265a39](https://github.com/standardnotes/server/commit/b265a39b635373c36ee8c3d8e09f0631159b3574))
+* logs verbosity during transitions ([e589029](https://github.com/standardnotes/server/commit/e589029722ab9f4debc8aa6cc78913f877eda2e3))
+
+## [1.35.5](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.35.4...@standardnotes/revisions-server@1.35.5) (2023-09-19)
+
+### Bug Fixes
+
+* add checking for secondary items logs ([a1a3e9f](https://github.com/standardnotes/server/commit/a1a3e9f586358b943b1b490a1382e42f081f7d06))
+* logs for removing already existing content and paging through diff of the content ([a40b17b](https://github.com/standardnotes/server/commit/a40b17b141f1d5954e1a45b969d5a941386c68d0))
+
+## [1.35.4](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.35.3...@standardnotes/revisions-server@1.35.4) (2023-09-19)
+
+### Bug Fixes
+
+* logs formatting during transition for better readability ([0ae028d](https://github.com/standardnotes/server/commit/0ae028db739decec8c50321b18b0af515e00bd23))
+
+## [1.35.3](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.35.2...@standardnotes/revisions-server@1.35.3) (2023-09-19)
+
+### Bug Fixes
+
+* **syncing-server:** paging through already existing items ([e4fcd73](https://github.com/standardnotes/server/commit/e4fcd738c35a4dc96e57db6ca08383a5647d61ad))
+
 ## [1.35.2](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.35.1...@standardnotes/revisions-server@1.35.2) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/revisions-server

packages/revisions/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/revisions-server",
-  "version": "1.35.2",
+  "version": "1.35.7",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/revisions/src/Domain/Handler/TransitionRequestedEventHandler.ts

@@ -29,7 +29,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
     }
 
     if (await this.isAlreadyMigrated(userUuid)) {
-      this.logger.info(`User ${event.payload.userUuid} already migrated.`)
+      this.logger.info(`[${event.payload.userUuid}] User already migrated.`)
 
       await this.domainEventPublisher.publish(
         this.domainEventFactory.createTransitionStatusUpdatedEvent({
@@ -43,7 +43,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
       return
     }
 
-    this.logger.info(`Handling transition requested event for user ${event.payload.userUuid}`)
+    this.logger.info(`[${event.payload.userUuid}] Handling transition requested event`)
 
     await this.domainEventPublisher.publish(
       this.domainEventFactory.createTransitionStatusUpdatedEvent({
@@ -59,7 +59,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
     })
 
     if (result.isFailed()) {
-      this.logger.error(`Failed to transition for user ${event.payload.userUuid}`)
+      this.logger.error(`[${event.payload.userUuid}] Failed to transition: ${result.getError()}`)
 
       await this.domainEventPublisher.publish(
         this.domainEventFactory.createTransitionStatusUpdatedEvent({
@@ -88,7 +88,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
 
     if (totalRevisionsCountForUserInPrimary > 0) {
       this.logger.info(
-        `User ${userUuid.value} has ${totalRevisionsCountForUserInPrimary} revisions in primary database.`,
+        `[${userUuid.value}] User has ${totalRevisionsCountForUserInPrimary} revisions in primary database.`,
       )
     }
 
@@ -98,9 +98,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
   private async getUserUuidFromEvent(event: TransitionRequestedEvent): Promise<Uuid | null> {
     const userUuidOrError = Uuid.create(event.payload.userUuid)
     if (userUuidOrError.isFailed()) {
-      this.logger.error(
+      this.logger.error(`[${event.payload.userUuid}] Failed to transition revisions: ${userUuidOrError.getError()}`)
-        `Failed to transition revisions for user ${event.payload.userUuid}: ${userUuidOrError.getError()}`,
-      )
       await this.domainEventPublisher.publish(
         this.domainEventFactory.createTransitionStatusUpdatedEvent({
           userUuid: event.payload.userUuid,

packages/revisions/src/Domain/UseCase/Transition/TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser/TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser.ts

@@ -17,7 +17,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
   ) {}
 
   async execute(dto: TransitionRevisionsFromPrimaryToSecondaryDatabaseForUserDTO): Promise<Result<void>> {
-    this.logger.info(`Transitioning revisions for user ${dto.userUuid}`)
+    this.logger.info(`[${dto.userUuid}] Transitioning revisions for user`)
 
     if (this.secondRevisionsRepository === null) {
       return Result.fail('Secondary revision repository is not set')
@@ -30,31 +30,28 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
     const userUuid = userUuidOrError.getValue()
 
     let newRevisionsInSecondaryCount = 0
-    let updatedRevisionsInSecondary: Revision[] = []
+    let updatedRevisionsInSecondary: string[] = []
+    let alreadyIdenticalInSecondaryAndPrimary: string[] = []
     if (await this.hasAlreadyDataInSecondaryDatabase(userUuid)) {
-      const { alreadyExistingInPrimary, newRevisionsInSecondary, updatedInSecondary } =
+      const { alreadyExistingInSecondaryAndPrimary, newRevisionsInSecondary, updatedInSecondary } =
         await this.getNewRevisionsCreatedInSecondaryDatabase(userUuid)
 
-      for (const existingRevision of alreadyExistingInPrimary) {
+      this.logger.info(
-        this.logger.info(`Removing revision ${existingRevision.id.toString()} from secondary database`)
+        `[${dto.userUuid}] ${alreadyExistingInSecondaryAndPrimary.length} already existing identical revisions in primary and secondary.`,
-        await (this.secondRevisionsRepository as RevisionRepositoryInterface).removeOneByUuid(
+      )
-          Uuid.create(existingRevision.id.toString()).getValue(),
+
-          userUuid,
+      alreadyIdenticalInSecondaryAndPrimary = alreadyExistingInSecondaryAndPrimary
-        )
-      }
 
       if (newRevisionsInSecondary.length > 0) {
         this.logger.info(
-          `Found ${newRevisionsInSecondary.length} new revisions in secondary database for user ${userUuid.value}`,
+          `[${dto.userUuid}] Found ${newRevisionsInSecondary.length} new revisions in secondary database`,
         )
       }
 
       newRevisionsInSecondaryCount = newRevisionsInSecondary.length
 
       if (updatedInSecondary.length > 0) {
-        this.logger.info(
+        this.logger.info(`[${dto.userUuid}] Found ${updatedInSecondary.length} updated revisions in secondary database`)
-          `Found ${updatedInSecondary.length} updated revisions in secondary database for user ${userUuid.value}`,
-        )
       }
 
       updatedRevisionsInSecondary = updatedInSecondary
@@ -66,15 +63,19 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
 
     const migrationTimeStart = this.timer.getTimestampInMicroseconds()
 
-    this.logger.debug(`Transitioning revisions for user ${userUuid.value}`)
+    this.logger.info(`[${dto.userUuid}] Migrating revisions`)
 
-    const migrationResult = await this.migrateRevisionsForUser(userUuid, updatedRevisionsInSecondary)
+    const migrationResult = await this.migrateRevisionsForUser(
+      userUuid,
+      updatedRevisionsInSecondary,
+      alreadyIdenticalInSecondaryAndPrimary,
+    )
     if (migrationResult.isFailed()) {
       if (newRevisionsInSecondaryCount === 0 && updatedRevisionsInSecondaryCount === 0) {
         const cleanupResult = await this.deleteRevisionsForUser(userUuid, this.secondRevisionsRepository)
         if (cleanupResult.isFailed()) {
           this.logger.error(
-            `Failed to clean up secondary database revisions for user ${userUuid.value}: ${cleanupResult.getError()}`,
+            `[${dto.userUuid}] Failed to clean up secondary database revisions: ${cleanupResult.getError()}`,
           )
         }
       }
@@ -88,13 +89,14 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
       userUuid,
       newRevisionsInSecondaryCount,
       updatedRevisionsInSecondary,
+      alreadyIdenticalInSecondaryAndPrimary,
     )
     if (integrityCheckResult.isFailed()) {
       if (newRevisionsInSecondaryCount === 0 && updatedRevisionsInSecondaryCount === 0) {
         const cleanupResult = await this.deleteRevisionsForUser(userUuid, this.secondRevisionsRepository)
         if (cleanupResult.isFailed()) {
           this.logger.error(
-            `Failed to clean up secondary database revisions for user ${userUuid.value}: ${cleanupResult.getError()}`,
+            `[${dto.userUuid}] Failed to clean up secondary database revisions: ${cleanupResult.getError()}`,
           )
         }
       }
@@ -104,9 +106,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
 
     const cleanupResult = await this.deleteRevisionsForUser(userUuid, this.primaryRevisionsRepository)
     if (cleanupResult.isFailed()) {
-      this.logger.error(
+      this.logger.error(`[${dto.userUuid}] Failed to clean up primary database revisions: ${cleanupResult.getError()}`)
-        `Failed to clean up primary database revisions for user ${userUuid.value}: ${cleanupResult.getError()}`,
-      )
     }
 
     const migrationTimeEnd = this.timer.getTimestampInMicroseconds()
@@ -115,7 +115,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
     const migrationDurationTimeStructure = this.timer.convertMicrosecondsToTimeStructure(migrationDuration)
 
     this.logger.info(
-      `Transitioned revisions for user ${userUuid.value} in ${migrationDurationTimeStructure.hours}h ${migrationDurationTimeStructure.minutes}m ${migrationDurationTimeStructure.seconds}s ${migrationDurationTimeStructure.milliseconds}ms`,
+      `[${dto.userUuid}] Transitioned revisions in ${migrationDurationTimeStructure.hours}h ${migrationDurationTimeStructure.minutes}m ${migrationDurationTimeStructure.seconds}s ${migrationDurationTimeStructure.milliseconds}ms`,
     )
 
     return Result.ok()
@@ -123,11 +123,11 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
 
   private async migrateRevisionsForUser(
     userUuid: Uuid,
-    updatedRevisionsInSecondary: Revision[],
+    updatedRevisionsInSecondary: string[],
+    alreadyExistingInSecondaryAndPrimary: string[],
   ): Promise<Result<void>> {
     try {
       const totalRevisionsCountForUser = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
-      let totalRevisionsCountTransitionedToSecondary = 0
       const totalPages = Math.ceil(totalRevisionsCountForUser / this.pageSize)
       for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
         const query = {
@@ -141,28 +141,29 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
         for (const revision of revisions) {
           try {
             if (
-              updatedRevisionsInSecondary.find(
+              updatedRevisionsInSecondary.find((updatedRevisionUuid) => updatedRevisionUuid === revision.id.toString())
-                (updatedRevision) => updatedRevision.id.toString() === revision.id.toString(),
-              )
             ) {
               this.logger.info(
-                `Skipping saving revision ${revision.id.toString()} as it was updated in secondary database`,
+                `[${
+                  userUuid.value
+                }] Skipping saving revision ${revision.id.toString()} as it was updated in secondary database`,
               )
 
               continue
             }
 
-            this.logger.debug(
+            if (
-              `Transitioning revision #${
+              alreadyExistingInSecondaryAndPrimary.find(
-                totalRevisionsCountTransitionedToSecondary + 1
+                (alreadyExistingRevisionUuid) => alreadyExistingRevisionUuid === revision.id.toString(),
-              }: ${revision.id.toString()} to secondary database`,
+              )
-            )
+            ) {
+              continue
+            }
 
             const didSave = await (this.secondRevisionsRepository as RevisionRepositoryInterface).insert(revision)
             if (!didSave) {
               return Result.fail(`Failed to save revision ${revision.id.toString()} to secondary database`)
             }
-            totalRevisionsCountTransitionedToSecondary++
           } catch (error) {
             return Result.fail(
               `Errored when saving revision ${revision.id.toString()} to secondary database: ${
@@ -173,8 +174,6 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
         }
       }
 
-      this.logger.debug(`Transitioned ${totalRevisionsCountTransitionedToSecondary} revisions to secondary database`)
-
       return Result.ok()
     } catch (error) {
       return Result.fail(`Errored when migrating revisions for user ${userUuid.value}: ${(error as Error).message}`)
@@ -195,8 +194,8 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
   }
 
   private async allowForSecondaryDatabaseToCatchUp(): Promise<void> {
-    const twoSecondsInMilliseconds = 2_000
+    const tenSecondsInMillisecondsToRebuildIndexes = 10_000
-    await this.timer.sleep(twoSecondsInMilliseconds)
+    await this.timer.sleep(tenSecondsInMillisecondsToRebuildIndexes)
   }
 
   private async hasAlreadyDataInSecondaryDatabase(userUuid: Uuid): Promise<boolean> {
@@ -207,7 +206,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
     const hasAlreadyDataInSecondaryDatabase = totalRevisionsCountForUserInSecondary > 0
     if (hasAlreadyDataInSecondaryDatabase) {
       this.logger.info(
-        `User ${userUuid.value} has already ${totalRevisionsCountForUserInSecondary} revisions in secondary database`,
+        `[${userUuid.value}] User has already ${totalRevisionsCountForUserInSecondary} revisions in secondary database`,
       )
     }
 
@@ -215,45 +214,57 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
   }
 
   private async getNewRevisionsCreatedInSecondaryDatabase(userUuid: Uuid): Promise<{
-    alreadyExistingInPrimary: Revision[]
+    alreadyExistingInSecondaryAndPrimary: string[]
-    newRevisionsInSecondary: Revision[]
+    newRevisionsInSecondary: string[]
-    updatedInSecondary: Revision[]
+    updatedInSecondary: string[]
   }> {
-    const revisions = await (this.secondRevisionsRepository as RevisionRepositoryInterface).findByUserUuid({
+    this.logger.info(`[${userUuid.value}] Checking for new revisions created in secondary database`)
-      userUuid: userUuid,
-    })
 
-    const alreadyExistingInPrimary: Revision[] = []
+    const totalRevisionsCountForUser = await (
-    const newRevisionsInSecondary: Revision[] = []
+      this.secondRevisionsRepository as RevisionRepositoryInterface
-    const updatedInSecondary: Revision[] = []
+    ).countByUserUuid(userUuid)
+    const totalPages = Math.ceil(totalRevisionsCountForUser / this.pageSize)
 
-    for (const revision of revisions) {
+    const alreadyExistingInSecondaryAndPrimary: string[] = []
-      const { revisionInPrimary, newerRevisionInSecondary } =
+    const newRevisionsInSecondary: string[] = []
-        await this.checkIfRevisionExistsInPrimaryDatabase(revision)
+    const updatedInSecondary: string[] = []
-      if (revisionInPrimary !== null) {
+
-        alreadyExistingInPrimary.push(revision)
+    for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
-        continue
+      const query = {
+        userUuid: userUuid,
+        offset: (currentPage - 1) * this.pageSize,
+        limit: this.pageSize,
       }
-      if (newerRevisionInSecondary !== null) {
+
-        updatedInSecondary.push(newerRevisionInSecondary)
+      const revisions = await (this.secondRevisionsRepository as RevisionRepositoryInterface).findByUserUuid(query)
-        continue
+      for (const revision of revisions) {
-      }
+        const { identicalRevisionInPrimary, newerRevisionInSecondary } =
-      if (revisionInPrimary === null && newerRevisionInSecondary === null) {
+          await this.checkIfRevisionExistsInPrimaryDatabase(revision)
-        newRevisionsInSecondary.push(revision)
+        if (identicalRevisionInPrimary !== null) {
-        continue
+          alreadyExistingInSecondaryAndPrimary.push(revision.id.toString())
+          continue
+        }
+        if (newerRevisionInSecondary !== null) {
+          updatedInSecondary.push(newerRevisionInSecondary.id.toString())
+          continue
+        }
+        if (identicalRevisionInPrimary === null && newerRevisionInSecondary === null) {
+          newRevisionsInSecondary.push(revision.id.toString())
+          continue
+        }
       }
     }
 
     return {
-      alreadyExistingInPrimary: alreadyExistingInPrimary,
+      alreadyExistingInSecondaryAndPrimary,
-      newRevisionsInSecondary: newRevisionsInSecondary,
+      newRevisionsInSecondary,
-      updatedInSecondary: updatedInSecondary,
+      updatedInSecondary,
     }
   }
 
   private async checkIfRevisionExistsInPrimaryDatabase(
     revision: Revision,
-  ): Promise<{ revisionInPrimary: Revision | null; newerRevisionInSecondary: Revision | null }> {
+  ): Promise<{ identicalRevisionInPrimary: Revision | null; newerRevisionInSecondary: Revision | null }> {
     const revisionInPrimary = await this.primaryRevisionsRepository.findOneByUuid(
       Uuid.create(revision.id.toString()).getValue(),
       revision.props.userUuid as Uuid,
@@ -262,27 +273,28 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
 
     if (revisionInPrimary === null) {
       return {
-        revisionInPrimary: null,
+        identicalRevisionInPrimary: null,
         newerRevisionInSecondary: null,
       }
     }
 
     if (!revision.isIdenticalTo(revisionInPrimary)) {
       this.logger.error(
-        `Revision ${revision.id.toString()} is not identical in primary and secondary database. Revision in secondary database: ${JSON.stringify(
+        `[${revision.props.userUuid
+          ?.value}] Revision ${revision.id.toString()} is not identical in primary and secondary database. Revision in secondary database: ${JSON.stringify(
           revision,
         )}, revision in primary database: ${JSON.stringify(revisionInPrimary)}`,
       )
 
       return {
-        revisionInPrimary: null,
+        identicalRevisionInPrimary: null,
         newerRevisionInSecondary:
           revision.props.dates.updatedAt > revisionInPrimary.props.dates.updatedAt ? revision : null,
       }
     }
 
     return {
-      revisionInPrimary: revisionInPrimary,
+      identicalRevisionInPrimary: revisionInPrimary,
       newerRevisionInSecondary: null,
     }
   }
@@ -290,7 +302,8 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
   private async checkIntegrityBetweenPrimaryAndSecondaryDatabase(
     userUuid: Uuid,
     newRevisionsInSecondaryCount: number,
-    updatedRevisionsInSecondary: Revision[],
+    updatedRevisionsInSecondary: string[],
+    alreadyExistingInSecondaryAndPrimary: string[],
   ): Promise<Result<boolean>> {
     try {
       const totalRevisionsCountForUserInPrimary = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
@@ -321,16 +334,24 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
           }
 
           if (
-            updatedRevisionsInSecondary.find(
+            updatedRevisionsInSecondary.find((updatedRevisionUuid) => updatedRevisionUuid === revision.id.toString())
-              (updatedRevision) => updatedRevision.id.toString() === revision.id.toString(),
-            )
           ) {
             this.logger.info(
-              `Skipping integrity check for revision ${revision.id.toString()} as it was updated in secondary database`,
+              `[${
+                userUuid.value
+              }] Skipping integrity check for revision ${revision.id.toString()} as it was updated in secondary database`,
             )
             continue
           }
 
+          if (
+            alreadyExistingInSecondaryAndPrimary.find(
+              (alreadyExistingRevisionUuid) => alreadyExistingRevisionUuid === revision.id.toString(),
+            )
+          ) {
+            continue
+          }
+
           if (!revision.isIdenticalTo(revisionInSecondary)) {
             return Result.fail(
               `Revision ${revision.id.toString()} is not identical in primary and secondary database. Revision in primary database: ${JSON.stringify(

packages/scheduler/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.20.49](https://github.com/standardnotes/server/compare/@standardnotes/scheduler-server@1.20.48...@standardnotes/scheduler-server@1.20.49) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/scheduler-server
+
 ## [1.20.48](https://github.com/standardnotes/server/compare/@standardnotes/scheduler-server@1.20.47...@standardnotes/scheduler-server@1.20.48) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/scheduler-server

packages/scheduler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/scheduler-server",
-  "version": "1.20.48",
+  "version": "1.20.49",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/settings/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.21.34](https://github.com/standardnotes/server/compare/@standardnotes/settings@1.21.33...@standardnotes/settings@1.21.34) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/settings
+
 ## [1.21.33](https://github.com/standardnotes/server/compare/@standardnotes/settings@1.21.32...@standardnotes/settings@1.21.33) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/settings

packages/settings/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/settings",
-  "version": "1.21.33",
+  "version": "1.21.34",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/syncing-server/CHANGELOG.md

@@ -3,6 +3,38 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.98.6](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.98.5...@standardnotes/syncing-server@1.98.6) (2023-09-19)
+
+### Bug Fixes
+
+* skip removing already existing content in secondary to pick up where the transition left of ([857c6af](https://github.com/standardnotes/syncing-server-js/commit/857c6af9468ec829ff4dce9a96ba7bf9c14d55a5))
+
+## [1.98.5](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.98.4...@standardnotes/syncing-server@1.98.5) (2023-09-19)
+
+### Bug Fixes
+
+* increase timeout for secondary database to catch up for indexes to be rebuilt ([b265a39](https://github.com/standardnotes/syncing-server-js/commit/b265a39b635373c36ee8c3d8e09f0631159b3574))
+* logs verbosity during transitions ([e589029](https://github.com/standardnotes/syncing-server-js/commit/e589029722ab9f4debc8aa6cc78913f877eda2e3))
+
+## [1.98.4](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.98.3...@standardnotes/syncing-server@1.98.4) (2023-09-19)
+
+### Bug Fixes
+
+* add checking for secondary items logs ([a1a3e9f](https://github.com/standardnotes/syncing-server-js/commit/a1a3e9f586358b943b1b490a1382e42f081f7d06))
+* logs for removing already existing content and paging through diff of the content ([a40b17b](https://github.com/standardnotes/syncing-server-js/commit/a40b17b141f1d5954e1a45b969d5a941386c68d0))
+
+## [1.98.3](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.98.2...@standardnotes/syncing-server@1.98.3) (2023-09-19)
+
+### Bug Fixes
+
+* logs formatting during transition for better readability ([0ae028d](https://github.com/standardnotes/syncing-server-js/commit/0ae028db739decec8c50321b18b0af515e00bd23))
+
+## [1.98.2](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.98.1...@standardnotes/syncing-server@1.98.2) (2023-09-19)
+
+### Bug Fixes
+
+* **syncing-server:** paging through already existing items ([e4fcd73](https://github.com/standardnotes/syncing-server-js/commit/e4fcd738c35a4dc96e57db6ca08383a5647d61ad))
+
 ## [1.98.1](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.98.0...@standardnotes/syncing-server@1.98.1) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/syncing-server

packages/syncing-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/syncing-server",
-  "version": "1.98.1",
+  "version": "1.98.6",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/syncing-server/src/Domain/Handler/TransitionRequestedEventHandler.ts

@@ -30,7 +30,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
     }
 
     if (await this.isAlreadyMigrated(userUuid)) {
-      this.logger.info(`User ${event.payload.userUuid} already migrated.`)
+      this.logger.info(`[${event.payload.userUuid}] User already migrated.`)
 
       await this.domainEventPublisher.publish(
         this.domainEventFactory.createTransitionStatusUpdatedEvent({
@@ -44,7 +44,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
       return
     }
 
-    this.logger.info(`Handling transition requested event for user ${event.payload.userUuid}`)
+    this.logger.info(`[${event.payload.userUuid}] Handling transition requested event`)
 
     await this.domainEventPublisher.publish(
       this.domainEventFactory.createTransitionStatusUpdatedEvent({
@@ -60,7 +60,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
     })
 
     if (result.isFailed()) {
-      this.logger.error(`Failed to trigger transition for user ${event.payload.userUuid}`)
+      this.logger.error(`[${event.payload.userUuid}] Failed to trigger transition: ${result.getError()}`)
 
       await this.domainEventPublisher.publish(
         this.domainEventFactory.createTransitionStatusUpdatedEvent({
@@ -90,7 +90,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
     })
 
     if (totalItemsCountForUserInPrimary > 0) {
-      this.logger.info(`User ${userUuid.value} has ${totalItemsCountForUserInPrimary} items in primary database.`)
+      this.logger.info(`[${userUuid.value}] User has ${totalItemsCountForUserInPrimary} items in primary database.`)
     }
 
     return totalItemsCountForUserInPrimary === 0
@@ -99,7 +99,7 @@ export class TransitionRequestedEventHandler implements DomainEventHandlerInterf
   private async getUserUuidFromEvent(event: TransitionRequestedEvent): Promise<Uuid | null> {
     const userUuidOrError = Uuid.create(event.payload.userUuid)
     if (userUuidOrError.isFailed()) {
-      this.logger.error(`Failed to transition items for user ${event.payload.userUuid}: ${userUuidOrError.getError()}`)
+      this.logger.error(`[${event.payload.userUuid}] Failed to transition items: ${userUuidOrError.getError()}`)
 
       await this.domainEventPublisher.publish(
         this.domainEventFactory.createTransitionStatusUpdatedEvent({

packages/syncing-server/src/Domain/Item/ItemRepositoryInterface.ts

@@ -15,6 +15,7 @@ export interface ItemRepositoryInterface {
   findByUuidAndUserUuid(uuid: string, userUuid: string): Promise<Item | null>
   findByUuid(uuid: Uuid): Promise<Item | null>
   remove(item: Item): Promise<void>
+  removeByUuid(uuid: Uuid): Promise<void>
   save(item: Item): Promise<void>
   markItemsAsDeleted(itemUuids: Array<string>, updatedAtTimestamp: number): Promise<void>
   updateContentSize(itemUuid: string, contentSize: number): Promise<void>

packages/syncing-server/src/Domain/UseCase/Transition/TransitionItemsFromPrimaryToSecondaryDatabaseForUser/TransitionItemsFromPrimaryToSecondaryDatabaseForUser.ts

@@ -18,7 +18,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
   ) {}
 
   async execute(dto: TransitionItemsFromPrimaryToSecondaryDatabaseForUserDTO): Promise<Result<void>> {
-    this.logger.info(`Transitioning items for user ${dto.userUuid}`)
+    this.logger.info(`[${dto.userUuid}] Transitioning items`)
 
     if (this.secondaryItemRepository === null) {
       return Result.fail('Secondary item repository is not set')
@@ -31,28 +31,26 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
     const userUuid = userUuidOrError.getValue()
 
     let newItemsInSecondaryCount = 0
-    let updatedItemsInSecondary: Item[] = []
+    let updatedItemsInSecondary: string[] = []
+    let alreadyIdenticalInSecondaryAndPrimary: string[] = []
     if (await this.hasAlreadyDataInSecondaryDatabase(userUuid)) {
-      const { alreadyExistingInPrimary, newItemsInSecondary, updatedInSecondary } =
+      const { alreadyExistingInSecondaryAndPrimary, newItemsInSecondary, updatedInSecondary } =
         await this.getNewItemsCreatedInSecondaryDatabase(userUuid)
 
-      for (const existingItem of alreadyExistingInPrimary) {
+      this.logger.info(
-        this.logger.info(`Removing item ${existingItem.uuid.value} from secondary database`)
+        `[${dto.userUuid}] ${alreadyExistingInSecondaryAndPrimary.length} already existing identical items in primary and secondary.`,
-        await (this.secondaryItemRepository as ItemRepositoryInterface).remove(existingItem)
+      )
-      }
+
+      alreadyIdenticalInSecondaryAndPrimary = alreadyExistingInSecondaryAndPrimary
 
       if (newItemsInSecondary.length > 0) {
-        this.logger.info(
+        this.logger.info(`[${dto.userUuid}] Found ${newItemsInSecondary.length} new items in secondary database.`)
-          `Found ${newItemsInSecondary.length} new items in secondary database for user ${userUuid.value}`,
-        )
       }
 
       newItemsInSecondaryCount = newItemsInSecondary.length
 
       if (updatedInSecondary.length > 0) {
-        this.logger.info(
+        this.logger.info(`[${dto.userUuid}] Found ${updatedInSecondary.length} updated items in secondary database.`)
-          `Found ${updatedInSecondary.length} updated items in secondary database for user ${userUuid.value}`,
-        )
       }
 
       updatedItemsInSecondary = updatedInSecondary
@@ -63,13 +61,19 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
 
     const migrationTimeStart = this.timer.getTimestampInMicroseconds()
 
-    const migrationResult = await this.migrateItemsForUser(userUuid, updatedItemsInSecondary)
+    this.logger.info(`[${dto.userUuid}] Migrating items`)
+
+    const migrationResult = await this.migrateItemsForUser(
+      userUuid,
+      updatedItemsInSecondary,
+      alreadyIdenticalInSecondaryAndPrimary,
+    )
     if (migrationResult.isFailed()) {
       if (newItemsInSecondaryCount === 0 && updatedItemsInSecondaryCount === 0) {
         const cleanupResult = await this.deleteItemsForUser(userUuid, this.secondaryItemRepository)
         if (cleanupResult.isFailed()) {
           this.logger.error(
-            `Failed to clean up secondary database items for user ${userUuid.value}: ${cleanupResult.getError()}`,
+            `[${dto.userUuid}] Failed to clean up secondary database items: ${cleanupResult.getError()}`,
           )
         }
       }
@@ -83,13 +87,14 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
       userUuid,
       newItemsInSecondaryCount,
       updatedItemsInSecondary,
+      alreadyIdenticalInSecondaryAndPrimary,
     )
     if (integrityCheckResult.isFailed()) {
       if (newItemsInSecondaryCount === 0 && updatedItemsInSecondaryCount === 0) {
         const cleanupResult = await this.deleteItemsForUser(userUuid, this.secondaryItemRepository)
         if (cleanupResult.isFailed()) {
           this.logger.error(
-            `Failed to clean up secondary database items for user ${userUuid.value}: ${cleanupResult.getError()}`,
+            `[${dto.userUuid}] Failed to clean up secondary database items: ${cleanupResult.getError()}`,
           )
         }
       }
@@ -99,9 +104,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
 
     const cleanupResult = await this.deleteItemsForUser(userUuid, this.primaryItemRepository)
     if (cleanupResult.isFailed()) {
-      this.logger.error(
+      this.logger.error(`[${dto.userUuid}] Failed to clean up primary database items: ${cleanupResult.getError()}`)
-        `Failed to clean up primary database items for user ${userUuid.value}: ${cleanupResult.getError()}`,
-      )
     }
 
     const migrationTimeEnd = this.timer.getTimestampInMicroseconds()
@@ -110,7 +113,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
     const migrationDurationTimeStructure = this.timer.convertMicrosecondsToTimeStructure(migrationDuration)
 
     this.logger.info(
-      `Transitioned items for user ${userUuid.value} in ${migrationDurationTimeStructure.hours}h ${migrationDurationTimeStructure.minutes}m ${migrationDurationTimeStructure.seconds}s ${migrationDurationTimeStructure.milliseconds}ms`,
+      `[${dto.userUuid}] Transitioned items in ${migrationDurationTimeStructure.hours}h ${migrationDurationTimeStructure.minutes}m ${migrationDurationTimeStructure.seconds}s ${migrationDurationTimeStructure.milliseconds}ms`,
     )
 
     return Result.ok()
@@ -123,48 +126,61 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
 
     const hasAlreadyDataInSecondaryDatabase = totalItemsCountForUser > 0
     if (hasAlreadyDataInSecondaryDatabase) {
-      this.logger.info(`User ${userUuid.value} has already ${totalItemsCountForUser} items in secondary database`)
+      this.logger.info(`[${userUuid.value}] User has already ${totalItemsCountForUser} items in secondary database`)
     }
 
     return hasAlreadyDataInSecondaryDatabase
   }
 
   private async allowForSecondaryDatabaseToCatchUp(): Promise<void> {
-    const twoSecondsInMilliseconds = 2_000
+    const tenSecondsInMillisecondsToRebuildIndexes = 10_000
-    await this.timer.sleep(twoSecondsInMilliseconds)
+    await this.timer.sleep(tenSecondsInMillisecondsToRebuildIndexes)
   }
 
   private async getNewItemsCreatedInSecondaryDatabase(userUuid: Uuid): Promise<{
-    alreadyExistingInPrimary: Item[]
+    alreadyExistingInSecondaryAndPrimary: string[]
-    newItemsInSecondary: Item[]
+    newItemsInSecondary: string[]
-    updatedInSecondary: Item[]
+    updatedInSecondary: string[]
   }> {
-    const items = await (this.secondaryItemRepository as ItemRepositoryInterface).findAll({
+    this.logger.info(`[${userUuid.value}] Checking for new items in secondary database`)
+
+    const alreadyExistingInSecondaryAndPrimary: string[] = []
+    const updatedInSecondary: string[] = []
+    const newItemsInSecondary: string[] = []
+
+    const totalItemsCountForUser = await (this.secondaryItemRepository as ItemRepositoryInterface).countAll({
       userUuid: userUuid.value,
     })
-
+    const totalPages = Math.ceil(totalItemsCountForUser / this.pageSize)
-    const alreadyExistingInPrimary: Item[] = []
+    for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
-    const updatedInSecondary: Item[] = []
+      const query: ItemQuery = {
-    const newItemsInSecondary: Item[] = []
+        userUuid: userUuid.value,
-
+        offset: (currentPage - 1) * this.pageSize,
-    for (const item of items) {
+        limit: this.pageSize,
-      const { itemInPrimary, newerItemInSecondary } = await this.checkIfItemExistsInPrimaryDatabase(item)
+        sortOrder: 'ASC',
-      if (itemInPrimary !== null) {
+        sortBy: 'uuid',
-        alreadyExistingInPrimary.push(item)
-        continue
       }
-      if (newerItemInSecondary !== null) {
+
-        updatedInSecondary.push(newerItemInSecondary)
+      const items = await (this.secondaryItemRepository as ItemRepositoryInterface).findAll(query)
-        continue
+      for (const item of items) {
-      }
+        const { identicalItemInPrimary, newerItemInSecondary } = await this.checkIfItemExistsInPrimaryDatabase(item)
-      if (itemInPrimary === null && newerItemInSecondary === null) {
+        if (identicalItemInPrimary !== null) {
-        newItemsInSecondary.push(item)
+          alreadyExistingInSecondaryAndPrimary.push(item.id.toString())
-        continue
+          continue
+        }
+        if (newerItemInSecondary !== null) {
+          updatedInSecondary.push(newerItemInSecondary.id.toString())
+          continue
+        }
+        if (identicalItemInPrimary === null && newerItemInSecondary === null) {
+          newItemsInSecondary.push(item.id.toString())
+          continue
+        }
       }
     }
 
     return {
-      alreadyExistingInPrimary,
+      alreadyExistingInSecondaryAndPrimary,
       newItemsInSecondary,
       updatedInSecondary,
     }
@@ -172,30 +188,36 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
 
   private async checkIfItemExistsInPrimaryDatabase(
     item: Item,
-  ): Promise<{ itemInPrimary: Item | null; newerItemInSecondary: Item | null }> {
+  ): Promise<{ identicalItemInPrimary: Item | null; newerItemInSecondary: Item | null }> {
     const itemInPrimary = await this.primaryItemRepository.findByUuid(item.uuid)
 
     if (itemInPrimary === null) {
-      return { itemInPrimary: null, newerItemInSecondary: null }
+      return { identicalItemInPrimary: null, newerItemInSecondary: null }
     }
 
     if (!item.isIdenticalTo(itemInPrimary)) {
       this.logger.error(
-        `Revision ${item.id.toString()} is not identical in primary and secondary database. Revision in secondary database: ${JSON.stringify(
+        `[${
+          item.props.userUuid.value
+        }] Item ${item.id.toString()} is not identical in primary and secondary database. Item in secondary database: ${JSON.stringify(
           item,
-        )}, revision in primary database: ${JSON.stringify(itemInPrimary)}`,
+        )}, item in primary database: ${JSON.stringify(itemInPrimary)}`,
       )
 
       return {
-        itemInPrimary: null,
+        identicalItemInPrimary: null,
         newerItemInSecondary: item.props.timestamps.updatedAt > itemInPrimary.props.timestamps.updatedAt ? item : null,
       }
     }
 
-    return { itemInPrimary: itemInPrimary, newerItemInSecondary: null }
+    return { identicalItemInPrimary: itemInPrimary, newerItemInSecondary: null }
   }
 
-  private async migrateItemsForUser(userUuid: Uuid, updatedItemsInSecondary: Item[]): Promise<Result<void>> {
+  private async migrateItemsForUser(
+    userUuid: Uuid,
+    updatedItemsInSecondary: string[],
+    alreadyExistingInSecondaryAndPrimary: string[],
+  ): Promise<Result<void>> {
     try {
       const totalItemsCountForUser = await this.primaryItemRepository.countAll({ userUuid: userUuid.value })
       const totalPages = Math.ceil(totalItemsCountForUser / this.pageSize)
@@ -204,16 +226,24 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
           userUuid: userUuid.value,
           offset: (currentPage - 1) * this.pageSize,
           limit: this.pageSize,
+          sortBy: 'uuid',
+          sortOrder: 'ASC',
         }
 
         const items = await this.primaryItemRepository.findAll(query)
 
         for (const item of items) {
-          if (updatedItemsInSecondary.find((updatedItem) => updatedItem.uuid.equals(item.uuid))) {
+          if (updatedItemsInSecondary.find((updatedItemUuid) => item.uuid.value === updatedItemUuid)) {
-            this.logger.info(`Skipping saving item ${item.uuid.value} as it was updated in secondary database`)
+            this.logger.info(
+              `[${userUuid.value}] Skipping saving item ${item.uuid.value} as it was updated in secondary database`,
+            )
 
             continue
           }
+          if (alreadyExistingInSecondaryAndPrimary.find((itemUuid) => item.uuid.value === itemUuid)) {
+            continue
+          }
+
           await (this.secondaryItemRepository as ItemRepositoryInterface).save(item)
         }
       }
@@ -237,7 +267,8 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
   private async checkIntegrityBetweenPrimaryAndSecondaryDatabase(
     userUuid: Uuid,
     newItemsInSecondaryCount: number,
-    updatedItemsInSecondary: Item[],
+    updatedItemsInSecondary: string[],
+    alreadyExistingInSecondaryAndPrimary: string[],
   ): Promise<Result<boolean>> {
     try {
       const totalItemsCountForUserInPrimary = await this.primaryItemRepository.countAll({ userUuid: userUuid.value })
@@ -259,6 +290,8 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
           userUuid: userUuid.value,
           offset: (currentPage - 1) * this.pageSize,
           limit: this.pageSize,
+          sortBy: 'uuid',
+          sortOrder: 'ASC',
         }
 
         const items = await this.primaryItemRepository.findAll(query)
@@ -269,13 +302,17 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
             return Result.fail(`Item ${item.uuid.value} not found in secondary database`)
           }
 
-          if (updatedItemsInSecondary.find((updatedItem) => updatedItem.uuid.equals(item.uuid))) {
+          if (updatedItemsInSecondary.find((updatedItemUuid) => item.uuid.value === updatedItemUuid)) {
             this.logger.info(
-              `Skipping integrity check for item ${item.uuid.value} as it was updated in secondary database`,
+              `[${userUuid.value}] Skipping integrity check for item ${item.uuid.value} as it was updated in secondary database`,
             )
             continue
           }
 
+          if (alreadyExistingInSecondaryAndPrimary.find((itemUuid) => item.uuid.value === itemUuid)) {
+            continue
+          }
+
           if (!item.isIdenticalTo(itemInSecondary)) {
             return Result.fail(
               `Item ${

packages/syncing-server/src/Infra/TypeORM/MongoDBItemRepository.ts

@@ -17,6 +17,10 @@ export class MongoDBItemRepository implements ItemRepositoryInterface {
     private logger: Logger,
   ) {}
 
+  async removeByUuid(uuid: Uuid): Promise<void> {
+    await this.mongoRepository.deleteOne({ _id: { $eq: BSON.UUID.createFromHexString(uuid.value) } })
+  }
+
   async deleteByUserUuid(userUuid: string): Promise<void> {
     await this.mongoRepository.deleteMany({ userUuid })
   }

packages/syncing-server/src/Infra/TypeORM/SQLLegacyItemRepository.ts

@@ -16,6 +16,15 @@ export class SQLLegacyItemRepository implements ItemRepositoryInterface {
     protected logger: Logger,
   ) {}
 
+  async removeByUuid(uuid: Uuid): Promise<void> {
+    await this.ormRepository
+      .createQueryBuilder('item')
+      .delete()
+      .from('items')
+      .where('uuid = :uuid', { uuid: uuid.value })
+      .execute()
+  }
+
   async save(item: Item): Promise<void> {
     const persistence = this.mapper.toProjection(item)
 

packages/websockets/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.10.46](https://github.com/standardnotes/server/compare/@standardnotes/websockets-server@1.10.45...@standardnotes/websockets-server@1.10.46) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/websockets-server
+
 ## [1.10.45](https://github.com/standardnotes/server/compare/@standardnotes/websockets-server@1.10.44...@standardnotes/websockets-server@1.10.45) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/websockets-server

packages/websockets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/websockets-server",
-  "version": "1.10.45",
+  "version": "1.10.46",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },