chore(release): publish new version

- @standardnotes/home-server@1.15.64
 - @standardnotes/revisions-server@1.35.3
 - @standardnotes/syncing-server@1.98.2

.github/workflows/common-docker-image.yml

@@ -33,17 +33,12 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Create Bundle Dir
-      id: bundle-dir
-      run: echo "temp_dir=$(mktemp -d -t ${{ inputs.service_name }}-${{ github.sha }}-XXXXXXX)" >> $GITHUB_OUTPUT
-
     - name: Cache build
       id: cache-build
       uses: actions/cache@v3
       with:
         path: |
           packages/**/dist
-          ${{ steps.bundle-dir.outputs.temp_dir }}
         key: ${{ runner.os }}-${{ inputs.service_name }}-build-${{ github.sha }}
 
     - name: Set up Node
@@ -57,11 +52,7 @@ jobs:
 
     - name: Build
       if: steps.cache-build.outputs.cache-hit != 'true'
-      run: yarn build ${{ inputs.package_path }}
-
-    - name: Bundle
-      if: steps.cache-build.outputs.cache-hit != 'true'
-      run: yarn workspace ${{ inputs.workspace_name }} bundle --no-compress --output-directory ${{ steps.bundle-dir.outputs.temp_dir }}
+      run: yarn build
 
     - name: Login to Docker Hub
       uses: docker/login-action@v2
@@ -93,8 +84,8 @@ jobs:
       uses: docker/build-push-action@v4
       with:
         builder: ${{ steps.buildx.outputs.name }}
-        context: ${{ steps.bundle-dir.outputs.temp_dir }}
-        file: ${{ steps.bundle-dir.outputs.temp_dir }}/${{ inputs.package_path }}/Dockerfile
+        context: .
+        file: ${{ inputs.package_path }}/Dockerfile
         platforms: linux/amd64,linux/arm64
         push: true
         tags: |

package.json

@@ -20,8 +20,8 @@
     "release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"",
     "publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose",
     "postversion": "./scripts/push-tags-one-by-one.sh",
-    "e2e": "yarn build packages/home-server && PORT=3123 yarn workspace @standardnotes/home-server start",
-    "start": "yarn build packages/home-server && yarn workspace @standardnotes/home-server start"
+    "e2e": "yarn workspace @standardnotes/home-server run build && PORT=3123 yarn workspace @standardnotes/home-server start",
+    "start": "yarn workspace @standardnotes/home-server run build && yarn workspace @standardnotes/home-server start"
   },
   "devDependencies": {
     "@commitlint/cli": "^17.0.2",

packages/auth/CHANGELOG.md

@@ -3,6 +3,18 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.143.6](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.143.4...@standardnotes/auth-server@1.143.6) (2023-09-19)
+
+### Bug Fixes
+
+* **auth:** bump version ([b91cd7e](https://github.com/standardnotes/server/commit/b91cd7e2329d68026665f61d9bd17ba918c57563))
+
+## [1.143.4](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.143.3...@standardnotes/auth-server@1.143.4) (2023-09-19)
+
+### Bug Fixes
+
+* **auth:** add debug logs to transition ([bf855bb](https://github.com/standardnotes/server/commit/bf855bb26e8a3618113bd2f5801c260782566815))
+
 ## [1.143.3](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.143.2...@standardnotes/auth-server@1.143.3) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/auth-server

packages/auth/bin/transition.ts

@@ -53,7 +53,10 @@ const requestTransition = async (
       continue
     }
 
+    let wasTransitionRequested = false
+
     if (itemsTransitionStatus?.value !== TransitionStatus.STATUSES.Verified) {
+      wasTransitionRequested = true
       await transitionStatusRepository.remove(user.uuid, 'items')
 
       await domainEventPublisher.publish(
@@ -66,6 +69,7 @@ const requestTransition = async (
     }
 
     if (revisionsTransitionStatus?.value !== TransitionStatus.STATUSES.Verified) {
+      wasTransitionRequested = true
       await transitionStatusRepository.remove(user.uuid, 'revisions')
 
       await domainEventPublisher.publish(
@@ -78,6 +82,12 @@ const requestTransition = async (
     }
 
     usersTriggered += 1
+
+    if (wasTransitionRequested) {
+      logger.info(
+        `[TRANSITION ${timestamp}] Transition requested for user ${user.uuid} - items status: ${itemsTransitionStatus?.value}, revisions status: ${revisionsTransitionStatus?.value}, has transition role: ${userHasTransitionRole}`,
+      )
+    }
   }
 
   logger.info(

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/auth-server",
-  "version": "1.143.3",
+  "version": "1.143.6",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/home-server/CHANGELOG.md

@@ -3,6 +3,18 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.15.64](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.63...@standardnotes/home-server@1.15.64) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.15.63](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.62...@standardnotes/home-server@1.15.63) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.15.62](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.61...@standardnotes/home-server@1.15.62) (2023-09-19)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
 ## [1.15.61](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.60...@standardnotes/home-server@1.15.61) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/home-server

packages/home-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/home-server",
-  "version": "1.15.61",
+  "version": "1.15.64",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/revisions/CHANGELOG.md

@@ -3,6 +3,12 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.35.3](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.35.2...@standardnotes/revisions-server@1.35.3) (2023-09-19)
+
+### Bug Fixes
+
+* **syncing-server:** paging through already existing items ([e4fcd73](https://github.com/standardnotes/server/commit/e4fcd738c35a4dc96e57db6ca08383a5647d61ad))
+
 ## [1.35.2](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.35.1...@standardnotes/revisions-server@1.35.2) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/revisions-server

packages/revisions/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/revisions-server",
-  "version": "1.35.2",
+  "version": "1.35.3",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/revisions/src/Domain/UseCase/Transition/TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser/TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser.ts

@@ -30,15 +30,15 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
     const userUuid = userUuidOrError.getValue()
 
     let newRevisionsInSecondaryCount = 0
-    let updatedRevisionsInSecondary: Revision[] = []
+    let updatedRevisionsInSecondary: string[] = []
     if (await this.hasAlreadyDataInSecondaryDatabase(userUuid)) {
       const { alreadyExistingInPrimary, newRevisionsInSecondary, updatedInSecondary } =
         await this.getNewRevisionsCreatedInSecondaryDatabase(userUuid)
 
-      for (const existingRevision of alreadyExistingInPrimary) {
-        this.logger.info(`Removing revision ${existingRevision.id.toString()} from secondary database`)
+      for (const existingRevisionUuid of alreadyExistingInPrimary) {
+        this.logger.info(`Removing revision ${existingRevisionUuid} from secondary database`)
         await (this.secondRevisionsRepository as RevisionRepositoryInterface).removeOneByUuid(
-          Uuid.create(existingRevision.id.toString()).getValue(),
+          Uuid.create(existingRevisionUuid).getValue(),
           userUuid,
         )
       }
@@ -121,10 +121,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
     return Result.ok()
   }
 
-  private async migrateRevisionsForUser(
-    userUuid: Uuid,
-    updatedRevisionsInSecondary: Revision[],
-  ): Promise<Result<void>> {
+  private async migrateRevisionsForUser(userUuid: Uuid, updatedRevisionsInSecondary: string[]): Promise<Result<void>> {
     try {
       const totalRevisionsCountForUser = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
       let totalRevisionsCountTransitionedToSecondary = 0
@@ -141,9 +138,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
         for (const revision of revisions) {
           try {
             if (
-              updatedRevisionsInSecondary.find(
-                (updatedRevision) => updatedRevision.id.toString() === revision.id.toString(),
-              )
+              updatedRevisionsInSecondary.find((updatedRevisionUuid) => updatedRevisionUuid === revision.id.toString())
             ) {
               this.logger.info(
                 `Skipping saving revision ${revision.id.toString()} as it was updated in secondary database`,
@@ -215,39 +210,47 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
   }
 
   private async getNewRevisionsCreatedInSecondaryDatabase(userUuid: Uuid): Promise<{
-    alreadyExistingInPrimary: Revision[]
-    newRevisionsInSecondary: Revision[]
-    updatedInSecondary: Revision[]
+    alreadyExistingInPrimary: string[]
+    newRevisionsInSecondary: string[]
+    updatedInSecondary: string[]
   }> {
-    const revisions = await (this.secondRevisionsRepository as RevisionRepositoryInterface).findByUserUuid({
-      userUuid: userUuid,
-    })
+    const totalRevisionsCountForUser = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
+    const totalPages = Math.ceil(totalRevisionsCountForUser / this.pageSize)
 
-    const alreadyExistingInPrimary: Revision[] = []
-    const newRevisionsInSecondary: Revision[] = []
-    const updatedInSecondary: Revision[] = []
+    const alreadyExistingInPrimary: string[] = []
+    const newRevisionsInSecondary: string[] = []
+    const updatedInSecondary: string[] = []
 
-    for (const revision of revisions) {
-      const { revisionInPrimary, newerRevisionInSecondary } =
-        await this.checkIfRevisionExistsInPrimaryDatabase(revision)
-      if (revisionInPrimary !== null) {
-        alreadyExistingInPrimary.push(revision)
-        continue
+    for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
+      const query = {
+        userUuid: userUuid,
+        offset: (currentPage - 1) * this.pageSize,
+        limit: this.pageSize,
       }
-      if (newerRevisionInSecondary !== null) {
-        updatedInSecondary.push(newerRevisionInSecondary)
-        continue
-      }
-      if (revisionInPrimary === null && newerRevisionInSecondary === null) {
-        newRevisionsInSecondary.push(revision)
-        continue
+
+      const revisions = await (this.secondRevisionsRepository as RevisionRepositoryInterface).findByUserUuid(query)
+      for (const revision of revisions) {
+        const { revisionInPrimary, newerRevisionInSecondary } =
+          await this.checkIfRevisionExistsInPrimaryDatabase(revision)
+        if (revisionInPrimary !== null) {
+          alreadyExistingInPrimary.push(revision.id.toString())
+          continue
+        }
+        if (newerRevisionInSecondary !== null) {
+          updatedInSecondary.push(newerRevisionInSecondary.id.toString())
+          continue
+        }
+        if (revisionInPrimary === null && newerRevisionInSecondary === null) {
+          newRevisionsInSecondary.push(revision.id.toString())
+          continue
+        }
       }
     }
 
     return {
-      alreadyExistingInPrimary: alreadyExistingInPrimary,
-      newRevisionsInSecondary: newRevisionsInSecondary,
-      updatedInSecondary: updatedInSecondary,
+      alreadyExistingInPrimary,
+      newRevisionsInSecondary,
+      updatedInSecondary,
     }
   }
 
@@ -290,7 +293,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
   private async checkIntegrityBetweenPrimaryAndSecondaryDatabase(
     userUuid: Uuid,
     newRevisionsInSecondaryCount: number,
-    updatedRevisionsInSecondary: Revision[],
+    updatedRevisionsInSecondary: string[],
   ): Promise<Result<boolean>> {
     try {
       const totalRevisionsCountForUserInPrimary = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
@@ -321,9 +324,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
           }
 
           if (
-            updatedRevisionsInSecondary.find(
-              (updatedRevision) => updatedRevision.id.toString() === revision.id.toString(),
-            )
+            updatedRevisionsInSecondary.find((updatedRevisionUuid) => updatedRevisionUuid === revision.id.toString())
           ) {
             this.logger.info(
               `Skipping integrity check for revision ${revision.id.toString()} as it was updated in secondary database`,

packages/syncing-server/CHANGELOG.md

@@ -3,6 +3,12 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.98.2](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.98.1...@standardnotes/syncing-server@1.98.2) (2023-09-19)
+
+### Bug Fixes
+
+* **syncing-server:** paging through already existing items ([e4fcd73](https://github.com/standardnotes/syncing-server-js/commit/e4fcd738c35a4dc96e57db6ca08383a5647d61ad))
+
 ## [1.98.1](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.98.0...@standardnotes/syncing-server@1.98.1) (2023-09-18)
 
 **Note:** Version bump only for package @standardnotes/syncing-server

packages/syncing-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/syncing-server",
-  "version": "1.98.1",
+  "version": "1.98.2",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/syncing-server/src/Domain/Item/ItemRepositoryInterface.ts

@@ -15,6 +15,7 @@ export interface ItemRepositoryInterface {
   findByUuidAndUserUuid(uuid: string, userUuid: string): Promise<Item | null>
   findByUuid(uuid: Uuid): Promise<Item | null>
   remove(item: Item): Promise<void>
+  removeByUuid(uuid: Uuid): Promise<void>
   save(item: Item): Promise<void>
   markItemsAsDeleted(itemUuids: Array<string>, updatedAtTimestamp: number): Promise<void>
   updateContentSize(itemUuid: string, contentSize: number): Promise<void>

packages/syncing-server/src/Domain/UseCase/Transition/TransitionItemsFromPrimaryToSecondaryDatabaseForUser/TransitionItemsFromPrimaryToSecondaryDatabaseForUser.ts

@@ -31,14 +31,16 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
     const userUuid = userUuidOrError.getValue()
 
     let newItemsInSecondaryCount = 0
-    let updatedItemsInSecondary: Item[] = []
+    let updatedItemsInSecondary: string[] = []
     if (await this.hasAlreadyDataInSecondaryDatabase(userUuid)) {
       const { alreadyExistingInPrimary, newItemsInSecondary, updatedInSecondary } =
         await this.getNewItemsCreatedInSecondaryDatabase(userUuid)
 
-      for (const existingItem of alreadyExistingInPrimary) {
-        this.logger.info(`Removing item ${existingItem.uuid.value} from secondary database`)
-        await (this.secondaryItemRepository as ItemRepositoryInterface).remove(existingItem)
+      for (const existingItemUuid of alreadyExistingInPrimary) {
+        this.logger.info(`Removing item ${existingItemUuid} from secondary database`)
+        await (this.secondaryItemRepository as ItemRepositoryInterface).removeByUuid(
+          Uuid.create(existingItemUuid).getValue(),
+        )
       }
 
       if (newItemsInSecondary.length > 0) {
@@ -135,31 +137,40 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
   }
 
   private async getNewItemsCreatedInSecondaryDatabase(userUuid: Uuid): Promise<{
-    alreadyExistingInPrimary: Item[]
-    newItemsInSecondary: Item[]
-    updatedInSecondary: Item[]
+    alreadyExistingInPrimary: string[]
+    newItemsInSecondary: string[]
+    updatedInSecondary: string[]
   }> {
-    const items = await (this.secondaryItemRepository as ItemRepositoryInterface).findAll({
-      userUuid: userUuid.value,
-    })
+    const alreadyExistingInPrimary: string[] = []
+    const updatedInSecondary: string[] = []
+    const newItemsInSecondary: string[] = []
 
-    const alreadyExistingInPrimary: Item[] = []
-    const updatedInSecondary: Item[] = []
-    const newItemsInSecondary: Item[] = []
+    const totalItemsCountForUser = await this.primaryItemRepository.countAll({ userUuid: userUuid.value })
+    const totalPages = Math.ceil(totalItemsCountForUser / this.pageSize)
+    for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
+      const query: ItemQuery = {
+        userUuid: userUuid.value,
+        offset: (currentPage - 1) * this.pageSize,
+        limit: this.pageSize,
+        sortOrder: 'ASC',
+        sortBy: 'uuid',
+      }
 
-    for (const item of items) {
-      const { itemInPrimary, newerItemInSecondary } = await this.checkIfItemExistsInPrimaryDatabase(item)
-      if (itemInPrimary !== null) {
-        alreadyExistingInPrimary.push(item)
-        continue
-      }
-      if (newerItemInSecondary !== null) {
-        updatedInSecondary.push(newerItemInSecondary)
-        continue
-      }
-      if (itemInPrimary === null && newerItemInSecondary === null) {
-        newItemsInSecondary.push(item)
-        continue
+      const items = await (this.secondaryItemRepository as ItemRepositoryInterface).findAll(query)
+      for (const item of items) {
+        const { itemInPrimary, newerItemInSecondary } = await this.checkIfItemExistsInPrimaryDatabase(item)
+        if (itemInPrimary !== null) {
+          alreadyExistingInPrimary.push(item.id.toString())
+          continue
+        }
+        if (newerItemInSecondary !== null) {
+          updatedInSecondary.push(newerItemInSecondary.id.toString())
+          continue
+        }
+        if (itemInPrimary === null && newerItemInSecondary === null) {
+          newItemsInSecondary.push(item.id.toString())
+          continue
+        }
       }
     }
 
@@ -195,7 +206,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
     return { itemInPrimary: itemInPrimary, newerItemInSecondary: null }
   }
 
-  private async migrateItemsForUser(userUuid: Uuid, updatedItemsInSecondary: Item[]): Promise<Result<void>> {
+  private async migrateItemsForUser(userUuid: Uuid, updatedItemsInSecondary: string[]): Promise<Result<void>> {
     try {
       const totalItemsCountForUser = await this.primaryItemRepository.countAll({ userUuid: userUuid.value })
       const totalPages = Math.ceil(totalItemsCountForUser / this.pageSize)
@@ -204,12 +215,14 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
           userUuid: userUuid.value,
           offset: (currentPage - 1) * this.pageSize,
           limit: this.pageSize,
+          sortBy: 'uuid',
+          sortOrder: 'ASC',
         }
 
         const items = await this.primaryItemRepository.findAll(query)
 
         for (const item of items) {
-          if (updatedItemsInSecondary.find((updatedItem) => updatedItem.uuid.equals(item.uuid))) {
+          if (updatedItemsInSecondary.find((updatedItemUuid) => item.uuid.value === updatedItemUuid)) {
             this.logger.info(`Skipping saving item ${item.uuid.value} as it was updated in secondary database`)
 
             continue
@@ -237,7 +250,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
   private async checkIntegrityBetweenPrimaryAndSecondaryDatabase(
     userUuid: Uuid,
     newItemsInSecondaryCount: number,
-    updatedItemsInSecondary: Item[],
+    updatedItemsInSecondary: string[],
   ): Promise<Result<boolean>> {
     try {
       const totalItemsCountForUserInPrimary = await this.primaryItemRepository.countAll({ userUuid: userUuid.value })
@@ -259,6 +272,8 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
           userUuid: userUuid.value,
           offset: (currentPage - 1) * this.pageSize,
           limit: this.pageSize,
+          sortBy: 'uuid',
+          sortOrder: 'ASC',
         }
 
         const items = await this.primaryItemRepository.findAll(query)
@@ -269,7 +284,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
             return Result.fail(`Item ${item.uuid.value} not found in secondary database`)
           }
 
-          if (updatedItemsInSecondary.find((updatedItem) => updatedItem.uuid.equals(item.uuid))) {
+          if (updatedItemsInSecondary.find((updatedItemUuid) => item.uuid.value === updatedItemUuid)) {
             this.logger.info(
               `Skipping integrity check for item ${item.uuid.value} as it was updated in secondary database`,
             )

packages/syncing-server/src/Infra/TypeORM/MongoDBItemRepository.ts

@@ -17,6 +17,10 @@ export class MongoDBItemRepository implements ItemRepositoryInterface {
     private logger: Logger,
   ) {}
 
+  async removeByUuid(uuid: Uuid): Promise<void> {
+    await this.mongoRepository.deleteOne({ _id: { $eq: BSON.UUID.createFromHexString(uuid.value) } })
+  }
+
   async deleteByUserUuid(userUuid: string): Promise<void> {
     await this.mongoRepository.deleteMany({ userUuid })
   }

packages/syncing-server/src/Infra/TypeORM/SQLLegacyItemRepository.ts

@@ -16,6 +16,15 @@ export class SQLLegacyItemRepository implements ItemRepositoryInterface {
     protected logger: Logger,
   ) {}
 
+  async removeByUuid(uuid: Uuid): Promise<void> {
+    await this.ormRepository
+      .createQueryBuilder('item')
+      .delete()
+      .from('items')
+      .where('uuid = :uuid', { uuid: uuid.value })
+      .execute()
+  }
+
   async save(item: Item): Promise<void> {
     const persistence = this.mapper.toProjection(item)