Compare commits

...

5 Commits

33 changed files with 209 additions and 42 deletions
+4
View File
@@ -3,6 +3,10 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [2.28.8](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.7...@standardnotes/analytics@2.28.8) (2023-10-10)
**Note:** Version bump only for package @standardnotes/analytics
## [2.28.7](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.6...@standardnotes/analytics@2.28.7) (2023-10-09) ## [2.28.7](https://github.com/standardnotes/server/compare/@standardnotes/analytics@2.28.6...@standardnotes/analytics@2.28.7) (2023-10-09)
**Note:** Version bump only for package @standardnotes/analytics **Note:** Version bump only for package @standardnotes/analytics
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/analytics", "name": "@standardnotes/analytics",
"version": "2.28.7", "version": "2.28.8",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
+4
View File
@@ -3,6 +3,10 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.78.5](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.78.4...@standardnotes/api-gateway@1.78.5) (2023-10-10)
**Note:** Version bump only for package @standardnotes/api-gateway
## [1.78.4](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.78.3...@standardnotes/api-gateway@1.78.4) (2023-10-09) ## [1.78.4](https://github.com/standardnotes/api-gateway/compare/@standardnotes/api-gateway@1.78.3...@standardnotes/api-gateway@1.78.4) (2023-10-09)
**Note:** Version bump only for package @standardnotes/api-gateway **Note:** Version bump only for package @standardnotes/api-gateway
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/api-gateway", "name": "@standardnotes/api-gateway",
"version": "1.78.4", "version": "1.78.5",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
+6
View File
@@ -3,6 +3,12 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
# [1.155.0](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.154.3...@standardnotes/auth-server@1.155.0) (2023-10-10)
### Features
* add more logs to transition ([783fd9e](https://github.com/standardnotes/server/commit/783fd9e2c6350a7f0f1e8d009d01a3328564aca0))
## [1.154.3](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.154.2...@standardnotes/auth-server@1.154.3) (2023-10-09) ## [1.154.3](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.154.2...@standardnotes/auth-server@1.154.3) (2023-10-09)
### Reverts ### Reverts
+12 -2
View File
@@ -17,6 +17,7 @@ import { RoleName, TransitionStatus } from '@standardnotes/domain-core'
const inputArgs = process.argv.slice(2) const inputArgs = process.argv.slice(2)
const startDateString = inputArgs[0] const startDateString = inputArgs[0]
const endDateString = inputArgs[1] const endDateString = inputArgs[1]
const forceRunParam = inputArgs[2]
const requestTransition = async ( const requestTransition = async (
transitionStatusRepository: TransitionStatusRepositoryInterface, transitionStatusRepository: TransitionStatusRepositoryInterface,
@@ -38,6 +39,7 @@ const requestTransition = async (
) )
let usersTriggered = 0 let usersTriggered = 0
const forceRun = forceRunParam === 'true'
for (const user of users) { for (const user of users) {
const itemsTransitionStatus = await transitionStatusRepository.getStatus(user.uuid, 'items') const itemsTransitionStatus = await transitionStatusRepository.getStatus(user.uuid, 'items')
const revisionsTransitionStatus = await transitionStatusRepository.getStatus(user.uuid, 'revisions') const revisionsTransitionStatus = await transitionStatusRepository.getStatus(user.uuid, 'revisions')
@@ -55,7 +57,11 @@ const requestTransition = async (
let wasTransitionRequested = false let wasTransitionRequested = false
if (itemsTransitionStatus === null || itemsTransitionStatus.value === TransitionStatus.STATUSES.Failed) { if (
itemsTransitionStatus === null ||
itemsTransitionStatus.value === TransitionStatus.STATUSES.Failed ||
(itemsTransitionStatus.value === TransitionStatus.STATUSES.InProgress && forceRun)
) {
wasTransitionRequested = true wasTransitionRequested = true
await transitionStatusRepository.remove(user.uuid, 'items') await transitionStatusRepository.remove(user.uuid, 'items')
@@ -68,7 +74,11 @@ const requestTransition = async (
) )
} }
if (revisionsTransitionStatus === null || revisionsTransitionStatus.value === TransitionStatus.STATUSES.Failed) { if (
revisionsTransitionStatus === null ||
revisionsTransitionStatus.value === TransitionStatus.STATUSES.Failed ||
(revisionsTransitionStatus.value === TransitionStatus.STATUSES.InProgress && forceRun)
) {
wasTransitionRequested = true wasTransitionRequested = true
await transitionStatusRepository.remove(user.uuid, 'revisions') await transitionStatusRepository.remove(user.uuid, 'revisions')
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/auth-server", "name": "@standardnotes/auth-server",
"version": "1.154.3", "version": "1.155.0",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
@@ -3,6 +3,12 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.16.4](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.16.3...@standardnotes/domain-events-infra@1.16.4) (2023-10-10)
### Bug Fixes
* **syncing-server:** add option to define otel ratio ([c021bb3](https://github.com/standardnotes/server/commit/c021bb3d7ca90179292e7c75f5a84bf2b941ce86))
## [1.16.3](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.16.2...@standardnotes/domain-events-infra@1.16.3) (2023-10-09) ## [1.16.3](https://github.com/standardnotes/server/compare/@standardnotes/domain-events-infra@1.16.2...@standardnotes/domain-events-infra@1.16.3) (2023-10-09)
### Bug Fixes ### Bug Fixes
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/domain-events-infra", "name": "@standardnotes/domain-events-infra",
"version": "1.16.3", "version": "1.16.4",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
@@ -17,7 +17,10 @@ import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express'
export class OpenTelemetrySDK implements OpenTelemetrySDKInterface { export class OpenTelemetrySDK implements OpenTelemetrySDKInterface {
private declare sdk: OpenTelemetrySDKNode.NodeSDK private declare sdk: OpenTelemetrySDKNode.NodeSDK
constructor(private serviceName: string) { constructor(
private serviceName: string,
private spanRatio?: number,
) {
this.build() this.build()
} }
@@ -41,8 +44,10 @@ export class OpenTelemetrySDK implements OpenTelemetrySDKInterface {
}, },
}) })
const ratio = this.spanRatio ?? 0.01
this.sdk = new OpenTelemetrySDKNode.NodeSDK({ this.sdk = new OpenTelemetrySDKNode.NodeSDK({
sampler: new OpenTelemetrySDKNode.tracing.TraceIdRatioBasedSampler(0.01), sampler: new OpenTelemetrySDKNode.tracing.TraceIdRatioBasedSampler(ratio),
textMapPropagator: new AWSXRayPropagator(), textMapPropagator: new AWSXRayPropagator(),
instrumentations: [ instrumentations: [
new HttpInstrumentation({ new HttpInstrumentation({
+4
View File
@@ -3,6 +3,10 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.12.18](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.17...@standardnotes/event-store@1.12.18) (2023-10-10)
**Note:** Version bump only for package @standardnotes/event-store
## [1.12.17](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.16...@standardnotes/event-store@1.12.17) (2023-10-09) ## [1.12.17](https://github.com/standardnotes/server/compare/@standardnotes/event-store@1.12.16...@standardnotes/event-store@1.12.17) (2023-10-09)
**Note:** Version bump only for package @standardnotes/event-store **Note:** Version bump only for package @standardnotes/event-store
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/event-store", "name": "@standardnotes/event-store",
"version": "1.12.17", "version": "1.12.18",
"description": "Event Store Service", "description": "Event Store Service",
"private": true, "private": true,
"main": "dist/src/index.js", "main": "dist/src/index.js",
+4
View File
@@ -3,6 +3,10 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.28.4](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.28.3...@standardnotes/files-server@1.28.4) (2023-10-10)
**Note:** Version bump only for package @standardnotes/files-server
## [1.28.3](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.28.2...@standardnotes/files-server@1.28.3) (2023-10-09) ## [1.28.3](https://github.com/standardnotes/files/compare/@standardnotes/files-server@1.28.2...@standardnotes/files-server@1.28.3) (2023-10-09)
### Reverts ### Reverts
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/files-server", "name": "@standardnotes/files-server",
"version": "1.28.3", "version": "1.28.4",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
+8
View File
@@ -3,6 +3,14 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.16.44](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.43...@standardnotes/home-server@1.16.44) (2023-10-10)
**Note:** Version bump only for package @standardnotes/home-server
## [1.16.43](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.42...@standardnotes/home-server@1.16.43) (2023-10-10)
**Note:** Version bump only for package @standardnotes/home-server
## [1.16.42](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.41...@standardnotes/home-server@1.16.42) (2023-10-09) ## [1.16.42](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.16.41...@standardnotes/home-server@1.16.42) (2023-10-09)
**Note:** Version bump only for package @standardnotes/home-server **Note:** Version bump only for package @standardnotes/home-server
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/home-server", "name": "@standardnotes/home-server",
"version": "1.16.42", "version": "1.16.44",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
+12
View File
@@ -3,6 +3,18 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.43.1](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.43.0...@standardnotes/revisions-server@1.43.1) (2023-10-10)
### Bug Fixes
* transition logs to be more verbose ([036317e](https://github.com/standardnotes/server/commit/036317e33347f121fa799cbd409f85759798369d))
# [1.43.0](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.42.3...@standardnotes/revisions-server@1.43.0) (2023-10-10)
### Features
* add more logs to transition ([783fd9e](https://github.com/standardnotes/server/commit/783fd9e2c6350a7f0f1e8d009d01a3328564aca0))
## [1.42.3](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.42.2...@standardnotes/revisions-server@1.42.3) (2023-10-09) ## [1.42.3](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.42.2...@standardnotes/revisions-server@1.42.3) (2023-10-09)
### Reverts ### Reverts
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/revisions-server", "name": "@standardnotes/revisions-server",
"version": "1.42.3", "version": "1.43.1",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
@@ -13,5 +13,6 @@ export interface RevisionRepositoryInterface {
updateUserUuid(itemUuid: Uuid, userUuid: Uuid): Promise<void> updateUserUuid(itemUuid: Uuid, userUuid: Uuid): Promise<void>
findByUserUuid(dto: { userUuid: Uuid; offset?: number; limit?: number }): Promise<Array<Revision>> findByUserUuid(dto: { userUuid: Uuid; offset?: number; limit?: number }): Promise<Array<Revision>>
insert(revision: Revision): Promise<boolean> insert(revision: Revision): Promise<boolean>
update(revision: Revision): Promise<boolean>
clearSharedVaultAndKeySystemAssociations(dto: { itemUuid?: Uuid; sharedVaultUuid: Uuid }): Promise<void> clearSharedVaultAndKeySystemAssociations(dto: { itemUuid?: Uuid; sharedVaultUuid: Uuid }): Promise<void>
} }
@@ -112,7 +112,13 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
const totalRevisionsCountForUser = await ( const totalRevisionsCountForUser = await (
this.secondRevisionsRepository as RevisionRepositoryInterface this.secondRevisionsRepository as RevisionRepositoryInterface
).countByUserUuid(userUuid) ).countByUserUuid(userUuid)
this.logger.info(`[${userUuid.value}] Total revisions count for user: ${totalRevisionsCountForUser}`)
const totalPages = Math.ceil(totalRevisionsCountForUser / this.pageSize) const totalPages = Math.ceil(totalRevisionsCountForUser / this.pageSize)
this.logger.info(`[${userUuid.value}] Total pages: ${totalPages}`)
let insertedCount = 0
let newerCount = 0
let identicalCount = 0
let updatedCount = 0
for (let currentPage = initialPage; currentPage <= totalPages; currentPage++) { for (let currentPage = initialPage; currentPage <= totalPages; currentPage++) {
const isPageInEvery10Percent = currentPage % Math.ceil(totalPages / 10) === 0 const isPageInEvery10Percent = currentPage % Math.ceil(totalPages / 10) === 0
if (isPageInEvery10Percent) { if (isPageInEvery10Percent) {
@@ -121,6 +127,9 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
(currentPage / totalPages) * 100, (currentPage / totalPages) * 100,
)}% completed`, )}% completed`,
) )
this.logger.info(
`[${userUuid.value}] Inserted ${insertedCount} revisions so far. Skipped ${newerCount} revisions because they were newer in primary database. Skipped ${identicalCount} revisions because they were identical in primary and secondary database. Updated ${updatedCount} revisions because they were older in primary database.`,
)
await this.updateTransitionStatus(userUuid, TransitionStatus.STATUSES.InProgress, timestamp) await this.updateTransitionStatus(userUuid, TransitionStatus.STATUSES.InProgress, timestamp)
} }
@@ -144,36 +153,31 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
[], [],
) )
if (revisionInPrimary !== null) { if (!revisionInPrimary) {
await this.primaryRevisionsRepository.insert(revision)
insertedCount++
} else {
if (revisionInPrimary.props.dates.updatedAt > revision.props.dates.updatedAt) { if (revisionInPrimary.props.dates.updatedAt > revision.props.dates.updatedAt) {
this.logger.info( this.logger.info(
`[${ `[${
userUuid.value userUuid.value
}] Revision ${revision.id.toString()} is older in secondary than revision in primary database`, }] Revision ${revision.id.toString()} is older in secondary than revision in primary database`,
) )
newerCount++
continue continue
} }
if (revisionInPrimary.isIdenticalTo(revision)) { if (revisionInPrimary.isIdenticalTo(revision)) {
identicalCount++
continue continue
} }
this.logger.info( await this.primaryRevisionsRepository.update(revision)
`[${userUuid.value}] Removing revision ${revision.id.toString()} in primary database: ${JSON.stringify(
revisionInPrimary,
)} as it is not identical to revision in secondary database: ${JSON.stringify(revision)}`,
)
await this.primaryRevisionsRepository.removeOneByUuid( updatedCount++
Uuid.create(revisionInPrimary.id.toString()).getValue(),
revisionInPrimary.props.userUuid as Uuid,
)
await this.allowForPrimaryDatabaseToCatchUp()
}
const didSave = await this.primaryRevisionsRepository.insert(revision)
if (!didSave) {
this.logger.error(`Failed to save revision ${revision.id.toString()} to primary database`)
} }
} catch (error) { } catch (error) {
this.logger.error( this.logger.error(
@@ -183,6 +187,10 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
} }
} }
this.logger.info(
`[${userUuid.value}] Inserted ${insertedCount} revisions. Skipped ${newerCount} revisions because they were newer in primary database. Skipped ${identicalCount} revisions because they were identical in primary and secondary database. Updated ${updatedCount} revisions because they were older in primary database.`,
)
return Result.ok() return Result.ok()
} catch (error) { } catch (error) {
return Result.fail(`Errored when migrating revisions for user ${userUuid.value}: ${(error as Error).message}`) return Result.fail(`Errored when migrating revisions for user ${userUuid.value}: ${(error as Error).message}`)
@@ -193,4 +193,14 @@ export class MongoDBRevisionRepository implements RevisionRepositoryInterface {
return insertResult.acknowledged return insertResult.acknowledged
} }
async update(revision: Revision): Promise<boolean> {
const persistence = this.revisionMapper.toProjection(revision)
const { _id, ...rest } = persistence
const updateResult = await this.mongoRepository.updateOne({ _id: _id }, { $set: rest })
return updateResult.acknowledged
}
} }
@@ -111,9 +111,19 @@ export class SQLLegacyRevisionRepository implements RevisionRepositoryInterface
} }
async insert(revision: Revision): Promise<boolean> { async insert(revision: Revision): Promise<boolean> {
const SQLLegacyRevision = this.revisionMapper.toProjection(revision) const projection = this.revisionMapper.toProjection(revision)
await this.ormRepository.insert(SQLLegacyRevision) await this.ormRepository.insert(projection)
return true
}
async update(revision: Revision): Promise<boolean> {
const projection = this.revisionMapper.toProjection(revision)
const { uuid, ...rest } = projection
await this.ormRepository.update({ uuid: uuid }, rest)
return true return true
} }
+4
View File
@@ -3,6 +3,10 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.22.8](https://github.com/standardnotes/server/compare/@standardnotes/scheduler-server@1.22.7...@standardnotes/scheduler-server@1.22.8) (2023-10-10)
**Note:** Version bump only for package @standardnotes/scheduler-server
## [1.22.7](https://github.com/standardnotes/server/compare/@standardnotes/scheduler-server@1.22.6...@standardnotes/scheduler-server@1.22.7) (2023-10-09) ## [1.22.7](https://github.com/standardnotes/server/compare/@standardnotes/scheduler-server@1.22.6...@standardnotes/scheduler-server@1.22.7) (2023-10-09)
**Note:** Version bump only for package @standardnotes/scheduler-server **Note:** Version bump only for package @standardnotes/scheduler-server
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/scheduler-server", "name": "@standardnotes/scheduler-server",
"version": "1.22.7", "version": "1.22.8",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
+16
View File
@@ -3,6 +3,22 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.115.1](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.115.0...@standardnotes/syncing-server@1.115.1) (2023-10-10)
### Bug Fixes
* transition logs to be more verbose ([036317e](https://github.com/standardnotes/syncing-server-js/commit/036317e33347f121fa799cbd409f85759798369d))
# [1.115.0](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.114.3...@standardnotes/syncing-server@1.115.0) (2023-10-10)
### Bug Fixes
* **syncing-server:** add option to define otel ratio ([c021bb3](https://github.com/standardnotes/syncing-server-js/commit/c021bb3d7ca90179292e7c75f5a84bf2b941ce86))
### Features
* add more logs to transition ([783fd9e](https://github.com/standardnotes/syncing-server-js/commit/783fd9e2c6350a7f0f1e8d009d01a3328564aca0))
## [1.114.3](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.114.2...@standardnotes/syncing-server@1.114.3) (2023-10-09) ## [1.114.3](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.114.2...@standardnotes/syncing-server@1.114.3) (2023-10-09)
### Bug Fixes ### Bug Fixes
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/syncing-server", "name": "@standardnotes/syncing-server",
"version": "1.114.3", "version": "1.115.1",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },
@@ -237,6 +237,7 @@ export class ContainerConfigLoader {
const isConfiguredForHomeServerOrSelfHosting = isConfiguredForHomeServer || isConfiguredForSelfHosting const isConfiguredForHomeServerOrSelfHosting = isConfiguredForHomeServer || isConfiguredForSelfHosting
const isSecondaryDatabaseEnabled = env.get('SECONDARY_DB_ENABLED', true) === 'true' const isSecondaryDatabaseEnabled = env.get('SECONDARY_DB_ENABLED', true) === 'true'
const isConfiguredForInMemoryCache = env.get('CACHE_TYPE', true) === 'memory' const isConfiguredForInMemoryCache = env.get('CACHE_TYPE', true) === 'memory'
const openTelemetryRatio = env.get('OTEL_SPAN_RATIO', true) ? +env.get('OTEL_SPAN_RATIO', true) : 0.01
container container
.bind<boolean>(TYPES.Sync_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING) .bind<boolean>(TYPES.Sync_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING)
@@ -250,6 +251,7 @@ export class ContainerConfigLoader {
this.mode === 'server' this.mode === 'server'
? ServiceIdentifier.NAMES.SyncingServer ? ServiceIdentifier.NAMES.SyncingServer
: ServiceIdentifier.NAMES.SyncingServerWorker, : ServiceIdentifier.NAMES.SyncingServerWorker,
openTelemetryRatio,
), ),
) )
} }
@@ -18,6 +18,8 @@ export interface ItemRepositoryInterface {
remove(item: Item): Promise<void> remove(item: Item): Promise<void>
removeByUuid(uuid: Uuid): Promise<void> removeByUuid(uuid: Uuid): Promise<void>
save(item: Item): Promise<void> save(item: Item): Promise<void>
insert(item: Item): Promise<void>
update(item: Item): Promise<void>
markItemsAsDeleted(itemUuids: Array<string>, updatedAtTimestamp: number): Promise<void> markItemsAsDeleted(itemUuids: Array<string>, updatedAtTimestamp: number): Promise<void>
updateContentSize(itemUuid: string, contentSize: number): Promise<void> updateContentSize(itemUuid: string, contentSize: number): Promise<void>
unassignFromSharedVault(sharedVaultUuid: Uuid): Promise<void> unassignFromSharedVault(sharedVaultUuid: Uuid): Promise<void>
@@ -114,13 +114,22 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
const totalItemsCountForUser = await (this.secondaryItemRepository as ItemRepositoryInterface).countAll({ const totalItemsCountForUser = await (this.secondaryItemRepository as ItemRepositoryInterface).countAll({
userUuid: userUuid.value, userUuid: userUuid.value,
}) })
this.logger.info(`[${userUuid.value}] Total items count for user: ${totalItemsCountForUser}`)
const totalPages = Math.ceil(totalItemsCountForUser / this.pageSize) const totalPages = Math.ceil(totalItemsCountForUser / this.pageSize)
this.logger.info(`[${userUuid.value}] Total pages: ${totalPages}`)
let insertedCount = 0
let updatedCount = 0
let newerCount = 0
let identicalCount = 0
for (let currentPage = initialPage; currentPage <= totalPages; currentPage++) { for (let currentPage = initialPage; currentPage <= totalPages; currentPage++) {
const isPageInEvery10Percent = currentPage % Math.ceil(totalPages / 10) === 0 const isPageInEvery10Percent = currentPage % Math.ceil(totalPages / 10) === 0
if (isPageInEvery10Percent) { if (isPageInEvery10Percent) {
this.logger.info( this.logger.info(
`[${userUuid.value}] Migrating items for user: ${Math.round((currentPage / totalPages) * 100)}% completed`, `[${userUuid.value}] Migrating items for user: ${Math.round((currentPage / totalPages) * 100)}% completed`,
) )
this.logger.info(
`[${userUuid.value}] Inserted items count: ${insertedCount}. Newer items count: ${newerCount}. Identical items count: ${identicalCount}. Updated items count: ${updatedCount}`,
)
await this.updateTransitionStatus(userUuid, TransitionStatus.STATUSES.InProgress, timestamp) await this.updateTransitionStatus(userUuid, TransitionStatus.STATUSES.InProgress, timestamp)
} }
@@ -143,28 +152,29 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
try { try {
const itemInPrimary = await this.primaryItemRepository.findByUuid(item.uuid) const itemInPrimary = await this.primaryItemRepository.findByUuid(item.uuid)
if (itemInPrimary !== null) { if (!itemInPrimary) {
await this.primaryItemRepository.insert(item)
insertedCount++
} else {
if (itemInPrimary.props.timestamps.updatedAt > item.props.timestamps.updatedAt) { if (itemInPrimary.props.timestamps.updatedAt > item.props.timestamps.updatedAt) {
this.logger.info( this.logger.info(
`[${userUuid.value}] Item ${item.uuid.value} is older in secondary than item in primary database`, `[${userUuid.value}] Item ${item.uuid.value} is older in secondary than item in primary database`,
) )
newerCount++
continue continue
} }
if (itemInPrimary.isIdenticalTo(item)) { if (itemInPrimary.isIdenticalTo(item)) {
identicalCount++
continue continue
} }
this.logger.info( await this.primaryItemRepository.update(item)
`[${userUuid.value}] Removing item ${item.uuid.value} in primary database as it is not identical to item in primary database`,
)
await this.primaryItemRepository.removeByUuid(item.uuid) updatedCount++
await this.allowForPrimaryDatabaseToCatchUp()
} }
await this.primaryItemRepository.save(item)
} catch (error) { } catch (error) {
this.logger.error( this.logger.error(
`Errored when saving item ${item.uuid.value} to primary database: ${(error as Error).message}`, `Errored when saving item ${item.uuid.value} to primary database: ${(error as Error).message}`,
@@ -173,6 +183,10 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
} }
} }
this.logger.info(
`[${userUuid.value}] Inserted items count: ${insertedCount}. Newer items count: ${newerCount}. Identical items count: ${identicalCount}. Updated items count: ${updatedCount}`,
)
return Result.ok() return Result.ok()
} catch (error) { } catch (error) {
return Result.fail((error as Error).message) return Result.fail((error as Error).message)
@@ -184,6 +184,25 @@ export class MongoDBItemRepository implements ItemRepositoryInterface {
) )
} }
async insert(item: Item): Promise<void> {
const persistence = this.mapper.toProjection(item)
await this.mongoRepository.insertOne(persistence)
}
async update(item: Item): Promise<void> {
const persistence = this.mapper.toProjection(item)
const { _id, ...rest } = persistence
await this.mongoRepository.updateOne(
{ _id: _id },
{
$set: rest,
},
)
}
async markItemsAsDeleted(itemUuids: string[], updatedAtTimestamp: number): Promise<void> { async markItemsAsDeleted(itemUuids: string[], updatedAtTimestamp: number): Promise<void> {
await this.mongoRepository.updateMany( await this.mongoRepository.updateMany(
{ _id: { $in: itemUuids.map((uuid) => BSON.UUID.createFromHexString(uuid)) } }, { _id: { $in: itemUuids.map((uuid) => BSON.UUID.createFromHexString(uuid)) } },
@@ -43,6 +43,20 @@ export class SQLLegacyItemRepository implements ItemRepositoryInterface {
await this.ormRepository.save(persistence) await this.ormRepository.save(persistence)
} }
async insert(item: Item): Promise<void> {
const projection = this.mapper.toProjection(item)
await this.ormRepository.insert(projection)
}
async update(item: Item): Promise<void> {
const projection = this.mapper.toProjection(item)
const { uuid, ...updateValues } = projection
await this.ormRepository.update({ uuid: uuid }, updateValues)
}
async remove(item: Item): Promise<void> { async remove(item: Item): Promise<void> {
await this.ormRepository.remove(this.mapper.toProjection(item)) await this.ormRepository.remove(this.mapper.toProjection(item))
} }
+4
View File
@@ -3,6 +3,10 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.14.4](https://github.com/standardnotes/server/compare/@standardnotes/websockets-server@1.14.3...@standardnotes/websockets-server@1.14.4) (2023-10-10)
**Note:** Version bump only for package @standardnotes/websockets-server
## [1.14.3](https://github.com/standardnotes/server/compare/@standardnotes/websockets-server@1.14.2...@standardnotes/websockets-server@1.14.3) (2023-10-09) ## [1.14.3](https://github.com/standardnotes/server/compare/@standardnotes/websockets-server@1.14.2...@standardnotes/websockets-server@1.14.3) (2023-10-09)
### Reverts ### Reverts
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@standardnotes/websockets-server", "name": "@standardnotes/websockets-server",
"version": "1.14.3", "version": "1.14.4",
"engines": { "engines": {
"node": ">=18.0.0 <21.0.0" "node": ">=18.0.0 <21.0.0"
}, },