Compare commits

..

8 Commits

42 changed files with 288 additions and 158 deletions

View File

@@ -3,6 +3,24 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.141.3](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.141.2...@standardnotes/auth-server@1.141.3) (2023-09-12)
### Bug Fixes
* transition adjustments ([f20a947](https://github.com/standardnotes/server/commit/f20a947f8a555c074d8dc1543c7a8bf61d1d887e))
## [1.141.2](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.141.1...@standardnotes/auth-server@1.141.2) (2023-09-11)
### Bug Fixes
* **auth:** remove transitioning upon sign out ([#819](https://github.com/standardnotes/server/issues/819)) ([330bff0](https://github.com/standardnotes/server/commit/330bff0124f5f49c3441304d166ea43c21fea7bc))
## [1.141.1](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.141.0...@standardnotes/auth-server@1.141.1) (2023-09-11)
### Bug Fixes
* disable running migrations in worker mode of a given service ([a82b9a0](https://github.com/standardnotes/server/commit/a82b9a0c8a023ba8a450ff9e34bcd62f928fcab3))
# [1.141.0](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.140.0...@standardnotes/auth-server@1.141.0) (2023-09-11)
### Features

View File

@@ -75,7 +75,7 @@ const requestBackups = async (
})
}
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
dayjs.extend(utc)

View File

@@ -18,7 +18,7 @@ const cleanup = async (
await cleanupExpiredSessions.execute({ date })
}
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
const env: Env = new Env()
env.load()

View File

@@ -8,7 +8,7 @@ import TYPES from '../src/Bootstrap/Types'
import { Env } from '../src/Bootstrap/Env'
import { PersistStatistics } from '../src/Domain/UseCase/PersistStatistics/PersistStatistics'
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
const env: Env = new Env()
env.load()

View File

@@ -10,6 +10,7 @@ import { Env } from '../src/Bootstrap/Env'
import { DomainEventPublisherInterface } from '@standardnotes/domain-events'
import { DomainEventFactoryInterface } from '../src/Domain/Event/DomainEventFactoryInterface'
import { UserRepositoryInterface } from '../src/Domain/User/UserRepositoryInterface'
import { RoleName } from '@standardnotes/domain-core'
const inputArgs = process.argv.slice(2)
const startDateString = inputArgs[0]
@@ -28,14 +29,27 @@ const requestTransition = async (
logger.info(`Found ${users.length} users created between ${startDateString} and ${endDateString}`)
let usersTriggered = 0
for (const user of users) {
const roles = await user.roles
const userHasTransitionUserRole = roles.some((role) => role.name === RoleName.NAMES.TransitionUser) === true
if (userHasTransitionUserRole === true) {
continue
}
const transitionRequestedEvent = domainEventFactory.createTransitionRequestedEvent({ userUuid: user.uuid })
usersTriggered += 1
await domainEventPublisher.publish(transitionRequestedEvent)
}
logger.info(
`Triggered transition for ${usersTriggered} users created between ${startDateString} and ${endDateString}`,
)
}
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
dayjs.extend(utc)

View File

@@ -63,7 +63,7 @@ const requestBackups = async (
return
}
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
dayjs.extend(utc)

View File

@@ -9,7 +9,7 @@ import { DomainEventSubscriberFactoryInterface } from '@standardnotes/domain-eve
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
dayjs.extend(utc)

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/auth-server",
"version": "1.141.0",
"version": "1.141.3",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -274,6 +274,8 @@ import { UserAddedToSharedVaultEventHandler } from '../Domain/Handler/UserAddedT
import { UserRemovedFromSharedVaultEventHandler } from '../Domain/Handler/UserRemovedFromSharedVaultEventHandler'
export class ContainerConfigLoader {
constructor(private mode: 'server' | 'worker' = 'server') {}
async load(configuration?: {
controllerConatiner?: ControllerContainerInterface
directCallDomainEventPublisher?: DirectCallDomainEventPublisher
@@ -310,7 +312,7 @@ export class ContainerConfigLoader {
}
container.bind<winston.Logger>(TYPES.Auth_Logger).toConstantValue(logger)
const appDataSource = new AppDataSource(env)
const appDataSource = new AppDataSource({ env, runMigrations: this.mode === 'server' })
await appDataSource.initialize()
logger.debug('Database initialized')
@@ -981,7 +983,6 @@ export class ContainerConfigLoader {
container.get(TYPES.Auth_GenerateRecoveryCodes),
container.get(TYPES.Auth_Logger),
container.get(TYPES.Auth_SessionService),
container.get(TYPES.Auth_TRANSITION_MODE_ENABLED),
),
)
container

View File

@@ -23,7 +23,12 @@ import { TypeORMSharedVaultUser } from '../Infra/TypeORM/TypeORMSharedVaultUser'
export class AppDataSource {
private _dataSource: DataSource | undefined
constructor(private env: Env) {}
constructor(
private configuration: {
env: Env
runMigrations: boolean
},
) {}
getRepository<Entity extends ObjectLiteral>(target: EntityTarget<Entity>): Repository<Entity> {
if (!this._dataSource) {
@@ -38,12 +43,12 @@ export class AppDataSource {
}
get dataSource(): DataSource {
this.env.load()
this.configuration.env.load()
const isConfiguredForMySQL = this.env.get('DB_TYPE') === 'mysql'
const isConfiguredForMySQL = this.configuration.env.get('DB_TYPE') === 'mysql'
const maxQueryExecutionTime = this.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
? +this.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
const maxQueryExecutionTime = this.configuration.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
? +this.configuration.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
: 45_000
const commonDataSourceOptions = {
@@ -68,28 +73,28 @@ export class AppDataSource {
TypeORMSharedVaultUser,
],
migrations: [`${__dirname}/../../migrations/${isConfiguredForMySQL ? 'mysql' : 'sqlite'}/*.js`],
migrationsRun: true,
logging: <LoggerOptions>this.env.get('DB_DEBUG_LEVEL', true) ?? 'info',
migrationsRun: this.configuration.runMigrations,
logging: <LoggerOptions>this.configuration.env.get('DB_DEBUG_LEVEL', true) ?? 'info',
}
if (isConfiguredForMySQL) {
const inReplicaMode = this.env.get('DB_REPLICA_HOST', true) ? true : false
const inReplicaMode = this.configuration.env.get('DB_REPLICA_HOST', true) ? true : false
const replicationConfig = {
master: {
host: this.env.get('DB_HOST'),
port: parseInt(this.env.get('DB_PORT')),
username: this.env.get('DB_USERNAME'),
password: this.env.get('DB_PASSWORD'),
database: this.env.get('DB_DATABASE'),
host: this.configuration.env.get('DB_HOST'),
port: parseInt(this.configuration.env.get('DB_PORT')),
username: this.configuration.env.get('DB_USERNAME'),
password: this.configuration.env.get('DB_PASSWORD'),
database: this.configuration.env.get('DB_DATABASE'),
},
slaves: [
{
host: this.env.get('DB_REPLICA_HOST', true),
port: parseInt(this.env.get('DB_PORT')),
username: this.env.get('DB_USERNAME'),
password: this.env.get('DB_PASSWORD'),
database: this.env.get('DB_DATABASE'),
host: this.configuration.env.get('DB_REPLICA_HOST', true),
port: parseInt(this.configuration.env.get('DB_PORT')),
username: this.configuration.env.get('DB_USERNAME'),
password: this.configuration.env.get('DB_PASSWORD'),
database: this.configuration.env.get('DB_DATABASE'),
},
],
removeNodeErrorCount: 10,
@@ -103,11 +108,11 @@ export class AppDataSource {
supportBigNumbers: true,
bigNumberStrings: false,
replication: inReplicaMode ? replicationConfig : undefined,
host: inReplicaMode ? undefined : this.env.get('DB_HOST'),
port: inReplicaMode ? undefined : parseInt(this.env.get('DB_PORT')),
username: inReplicaMode ? undefined : this.env.get('DB_USERNAME'),
password: inReplicaMode ? undefined : this.env.get('DB_PASSWORD'),
database: inReplicaMode ? undefined : this.env.get('DB_DATABASE'),
host: inReplicaMode ? undefined : this.configuration.env.get('DB_HOST'),
port: inReplicaMode ? undefined : parseInt(this.configuration.env.get('DB_PORT')),
username: inReplicaMode ? undefined : this.configuration.env.get('DB_USERNAME'),
password: inReplicaMode ? undefined : this.configuration.env.get('DB_PASSWORD'),
database: inReplicaMode ? undefined : this.configuration.env.get('DB_DATABASE'),
}
this._dataSource = new DataSource(mySQLDataSourceOptions)
@@ -115,7 +120,7 @@ export class AppDataSource {
const sqliteDataSourceOptions: SqliteConnectionOptions = {
...commonDataSourceOptions,
type: 'sqlite',
database: this.env.get('DB_SQLITE_DATABASE_PATH'),
database: this.configuration.env.get('DB_SQLITE_DATABASE_PATH'),
enableWAL: true,
busyErrorRetry: 2000,
}

View File

@@ -4,4 +4,4 @@ import { Env } from './Env'
const env: Env = new Env()
env.load()
export const MigrationsDataSource = new AppDataSource(env).dataSource
export const MigrationsDataSource = new AppDataSource({ env, runMigrations: true }).dataSource

View File

@@ -27,7 +27,6 @@ describe('AuthController', () => {
let doGenerateRecoveryCodes: GenerateRecoveryCodes
let logger: Logger
let sessionService: SessionServiceInterface
let transitionModeEnabled: boolean
const createController = () =>
new AuthController(
@@ -40,7 +39,6 @@ describe('AuthController', () => {
doGenerateRecoveryCodes,
logger,
sessionService,
transitionModeEnabled,
)
beforeEach(() => {
@@ -66,8 +64,6 @@ describe('AuthController', () => {
sessionService = {} as jest.Mocked<SessionServiceInterface>
sessionService.deleteSessionByToken = jest.fn().mockReturnValue('1-2-3')
transitionModeEnabled = false
})
it('should register a user', async () => {

View File

@@ -37,7 +37,6 @@ export class AuthController implements UserServerInterface {
private doGenerateRecoveryCodes: GenerateRecoveryCodes,
private logger: Logger,
private sessionService: SessionServiceInterface,
private transitionModeEnabled: boolean,
) {}
async update(_params: UserUpdateRequestParams): Promise<HttpResponse<UserUpdateResponse>> {
@@ -227,14 +226,6 @@ export class AuthController implements UserServerInterface {
let headers = undefined
if (userUuid !== null) {
headers = new Map([['x-invalidate-cache', userUuid]])
if (this.transitionModeEnabled) {
await this.domainEventPublisher.publish(
this.domainEventFactory.createTransitionRequestedEvent({
userUuid,
}),
)
}
}
return {

View File

@@ -1,5 +1,8 @@
export interface TransitionStatusRepositoryInterface {
updateStatus(userUuid: string, transitionType: 'items' | 'revisions', status: 'STARTED' | 'FAILED'): Promise<void>
removeStatus(userUuid: string, transitionType: 'items' | 'revisions'): Promise<void>
getStatus(userUuid: string, transitionType: 'items' | 'revisions'): Promise<'STARTED' | 'FAILED' | null>
getStatus(
userUuid: string,
transitionType: 'items' | 'revisions',
): Promise<'STARTED' | 'IN_PROGRESS' | 'FAILED' | null>
}

View File

@@ -119,7 +119,7 @@ describe('CreateCrossServiceToken', () => {
})
it('should create a cross service token for user that has an ongoing transaction', async () => {
transitionStatusRepository.getStatus = jest.fn().mockReturnValue('STARTED')
transitionStatusRepository.getStatus = jest.fn().mockReturnValue('IN_PROGRESS')
await createUseCase().execute({
user,

View File

@@ -59,7 +59,7 @@ export class CreateCrossServiceToken implements UseCaseInterface<string> {
user: this.projectUser(user),
roles: this.projectRoles(roles),
shared_vault_owner_context: undefined,
ongoing_transition: transitionStatus === 'STARTED',
ongoing_transition: transitionStatus === 'IN_PROGRESS',
belongs_to_shared_vaults: sharedVaultAssociations.map((association) => ({
shared_vault_uuid: association.props.sharedVaultUuid.value,
permission: association.props.permission.value,

View File

@@ -4,13 +4,17 @@ import { GetTransitionStatusDTO } from './GetTransitionStatusDTO'
import { UserRepositoryInterface } from '../../User/UserRepositoryInterface'
import { TransitionStatusRepositoryInterface } from '../../Transition/TransitionStatusRepositoryInterface'
export class GetTransitionStatus implements UseCaseInterface<'TO-DO' | 'STARTED' | 'FINISHED' | 'FAILED'> {
export class GetTransitionStatus
implements UseCaseInterface<'TO-DO' | 'STARTED' | 'IN_PROGRESS' | 'FINISHED' | 'FAILED'>
{
constructor(
private transitionStatusRepository: TransitionStatusRepositoryInterface,
private userRepository: UserRepositoryInterface,
) {}
async execute(dto: GetTransitionStatusDTO): Promise<Result<'TO-DO' | 'STARTED' | 'FINISHED' | 'FAILED'>> {
async execute(
dto: GetTransitionStatusDTO,
): Promise<Result<'TO-DO' | 'STARTED' | 'IN_PROGRESS' | 'FINISHED' | 'FAILED'>> {
const userUuidOrError = Uuid.create(dto.userUuid)
if (userUuidOrError.isFailed()) {
return Result.fail(userUuidOrError.getError())

View File

@@ -19,9 +19,13 @@ export class RedisTransitionStatusRepository implements TransitionStatusReposito
await this.redisClient.del(`${this.PREFIX}:${transitionType}:${userUuid}`)
}
async getStatus(userUuid: string, transitionType: 'items' | 'revisions'): Promise<'STARTED' | 'FAILED' | null> {
async getStatus(
userUuid: string,
transitionType: 'items' | 'revisions',
): Promise<'STARTED' | 'IN_PROGRESS' | 'FAILED' | null> {
const status = (await this.redisClient.get(`${this.PREFIX}:${transitionType}:${userUuid}`)) as
| 'STARTED'
| 'IN_PROGRESS'
| 'FAILED'
| null

View File

@@ -3,6 +3,22 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.15.37](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.36...@standardnotes/home-server@1.15.37) (2023-09-12)
**Note:** Version bump only for package @standardnotes/home-server
## [1.15.36](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.35...@standardnotes/home-server@1.15.36) (2023-09-11)
**Note:** Version bump only for package @standardnotes/home-server
## [1.15.35](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.34...@standardnotes/home-server@1.15.35) (2023-09-11)
**Note:** Version bump only for package @standardnotes/home-server
## [1.15.34](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.33...@standardnotes/home-server@1.15.34) (2023-09-11)
**Note:** Version bump only for package @standardnotes/home-server
## [1.15.33](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.32...@standardnotes/home-server@1.15.33) (2023-09-11)
**Note:** Version bump only for package @standardnotes/home-server

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/home-server",
"version": "1.15.33",
"version": "1.15.37",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -3,6 +3,24 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.33.8](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.33.7...@standardnotes/revisions-server@1.33.8) (2023-09-12)
### Bug Fixes
* transition adjustments ([f20a947](https://github.com/standardnotes/server/commit/f20a947f8a555c074d8dc1543c7a8bf61d1d887e))
## [1.33.7](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.33.6...@standardnotes/revisions-server@1.33.7) (2023-09-11)
### Bug Fixes
* **auth:** remove transitioning upon sign out ([#819](https://github.com/standardnotes/server/issues/819)) ([330bff0](https://github.com/standardnotes/server/commit/330bff0124f5f49c3441304d166ea43c21fea7bc))
## [1.33.6](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.33.5...@standardnotes/revisions-server@1.33.6) (2023-09-11)
### Bug Fixes
* disable running migrations in worker mode of a given service ([a82b9a0](https://github.com/standardnotes/server/commit/a82b9a0c8a023ba8a450ff9e34bcd62f928fcab3))
## [1.33.5](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.33.4...@standardnotes/revisions-server@1.33.5) (2023-09-11)
### Bug Fixes

View File

@@ -7,7 +7,7 @@ import { Env } from '../src/Bootstrap/Env'
import { DomainEventSubscriberFactoryInterface } from '@standardnotes/domain-events'
import { ContainerConfigLoader } from '../src/Bootstrap/Container'
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
const env: Env = new Env()
env.load()

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/revisions-server",
"version": "1.33.5",
"version": "1.33.8",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -70,6 +70,8 @@ import { RemoveRevisionsFromSharedVault } from '../Domain/UseCase/RemoveRevision
import { ItemRemovedFromSharedVaultEventHandler } from '../Domain/Handler/ItemRemovedFromSharedVaultEventHandler'
export class ContainerConfigLoader {
constructor(private mode: 'server' | 'worker' = 'server') {}
async load(configuration?: {
controllerConatiner?: ControllerContainerInterface
directCallDomainEventPublisher?: DirectCallDomainEventPublisher
@@ -115,7 +117,7 @@ export class ContainerConfigLoader {
container.bind<TimerInterface>(TYPES.Revisions_Timer).toDynamicValue(() => new Timer())
const appDataSource = new AppDataSource(env)
const appDataSource = new AppDataSource({ env, runMigrations: this.mode === 'server' })
await appDataSource.initialize()
logger.debug('Database initialized')
@@ -348,6 +350,7 @@ export class ContainerConfigLoader {
: null,
container.get<TimerInterface>(TYPES.Revisions_Timer),
container.get<winston.Logger>(TYPES.Revisions_Logger),
env.get('MIGRATION_BATCH_SIZE', true) ? +env.get('MIGRATION_BATCH_SIZE', true) : 100,
),
)
container

View File

@@ -12,7 +12,12 @@ export class AppDataSource {
private _dataSource: DataSource | undefined
private _secondaryDataSource: DataSource | undefined
constructor(private env: Env) {}
constructor(
private configuration: {
env: Env
runMigrations: boolean
},
) {}
getRepository<Entity extends ObjectLiteral>(target: EntityTarget<Entity>): Repository<Entity> {
if (!this._dataSource) {
@@ -39,20 +44,20 @@ export class AppDataSource {
}
get secondaryDataSource(): DataSource | undefined {
this.env.load()
this.configuration.env.load()
if (this.env.get('SECONDARY_DB_ENABLED', true) !== 'true') {
if (this.configuration.env.get('SECONDARY_DB_ENABLED', true) !== 'true') {
return undefined
}
this._secondaryDataSource = new DataSource({
type: 'mongodb',
host: this.env.get('MONGO_HOST'),
host: this.configuration.env.get('MONGO_HOST'),
authSource: 'admin',
port: parseInt(this.env.get('MONGO_PORT')),
username: this.env.get('MONGO_USERNAME'),
password: this.env.get('MONGO_PASSWORD', true),
database: this.env.get('MONGO_DATABASE'),
port: parseInt(this.configuration.env.get('MONGO_PORT')),
username: this.configuration.env.get('MONGO_USERNAME'),
password: this.configuration.env.get('MONGO_PASSWORD', true),
database: this.configuration.env.get('MONGO_DATABASE'),
entities: [MongoDBRevision],
retryWrites: false,
synchronize: true,
@@ -62,15 +67,16 @@ export class AppDataSource {
}
get dataSource(): DataSource {
this.env.load()
this.configuration.env.load()
const isConfiguredForMySQL = this.env.get('DB_TYPE') === 'mysql'
const isConfiguredForMySQL = this.configuration.env.get('DB_TYPE') === 'mysql'
const isConfiguredForHomeServerOrSelfHosting =
this.env.get('MODE', true) === 'home-server' || this.env.get('MODE', true) === 'self-hosted'
this.configuration.env.get('MODE', true) === 'home-server' ||
this.configuration.env.get('MODE', true) === 'self-hosted'
const maxQueryExecutionTime = this.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
? +this.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
const maxQueryExecutionTime = this.configuration.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
? +this.configuration.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
: 45_000
const migrationsSourceDirectoryName = isConfiguredForMySQL
@@ -83,28 +89,28 @@ export class AppDataSource {
maxQueryExecutionTime,
entities: [isConfiguredForHomeServerOrSelfHosting ? SQLRevision : SQLLegacyRevision],
migrations: [`${__dirname}/../../migrations/${migrationsSourceDirectoryName}/*.js`],
migrationsRun: true,
logging: <LoggerOptions>this.env.get('DB_DEBUG_LEVEL', true) ?? 'info',
migrationsRun: this.configuration.runMigrations,
logging: <LoggerOptions>this.configuration.env.get('DB_DEBUG_LEVEL', true) ?? 'info',
}
if (isConfiguredForMySQL) {
const inReplicaMode = this.env.get('DB_REPLICA_HOST', true) ? true : false
const inReplicaMode = this.configuration.env.get('DB_REPLICA_HOST', true) ? true : false
const replicationConfig = {
master: {
host: this.env.get('DB_HOST'),
port: parseInt(this.env.get('DB_PORT')),
username: this.env.get('DB_USERNAME'),
password: this.env.get('DB_PASSWORD'),
database: this.env.get('DB_DATABASE'),
host: this.configuration.env.get('DB_HOST'),
port: parseInt(this.configuration.env.get('DB_PORT')),
username: this.configuration.env.get('DB_USERNAME'),
password: this.configuration.env.get('DB_PASSWORD'),
database: this.configuration.env.get('DB_DATABASE'),
},
slaves: [
{
host: this.env.get('DB_REPLICA_HOST', true),
port: parseInt(this.env.get('DB_PORT')),
username: this.env.get('DB_USERNAME'),
password: this.env.get('DB_PASSWORD'),
database: this.env.get('DB_DATABASE'),
host: this.configuration.env.get('DB_REPLICA_HOST', true),
port: parseInt(this.configuration.env.get('DB_PORT')),
username: this.configuration.env.get('DB_USERNAME'),
password: this.configuration.env.get('DB_PASSWORD'),
database: this.configuration.env.get('DB_DATABASE'),
},
],
removeNodeErrorCount: 10,
@@ -118,11 +124,11 @@ export class AppDataSource {
supportBigNumbers: true,
bigNumberStrings: false,
replication: inReplicaMode ? replicationConfig : undefined,
host: inReplicaMode ? undefined : this.env.get('DB_HOST'),
port: inReplicaMode ? undefined : parseInt(this.env.get('DB_PORT')),
username: inReplicaMode ? undefined : this.env.get('DB_USERNAME'),
password: inReplicaMode ? undefined : this.env.get('DB_PASSWORD'),
database: inReplicaMode ? undefined : this.env.get('DB_DATABASE'),
host: inReplicaMode ? undefined : this.configuration.env.get('DB_HOST'),
port: inReplicaMode ? undefined : parseInt(this.configuration.env.get('DB_PORT')),
username: inReplicaMode ? undefined : this.configuration.env.get('DB_USERNAME'),
password: inReplicaMode ? undefined : this.configuration.env.get('DB_PASSWORD'),
database: inReplicaMode ? undefined : this.configuration.env.get('DB_DATABASE'),
}
this._dataSource = new DataSource(mySQLDataSourceOptions)
@@ -130,7 +136,7 @@ export class AppDataSource {
const sqliteDataSourceOptions: SqliteConnectionOptions = {
...commonDataSourceOptions,
type: 'sqlite',
database: this.env.get('DB_SQLITE_DATABASE_PATH'),
database: this.configuration.env.get('DB_SQLITE_DATABASE_PATH'),
enableWAL: true,
busyErrorRetry: 2000,
}

View File

@@ -4,4 +4,4 @@ import { Env } from './Env'
const env: Env = new Env()
env.load()
export const MigrationsDataSource = new AppDataSource(env).dataSource
export const MigrationsDataSource = new AppDataSource({ env, runMigrations: true }).dataSource

View File

@@ -4,6 +4,6 @@ export interface DomainEventFactoryInterface {
createTransitionStatusUpdatedEvent(dto: {
userUuid: string
transitionType: 'items' | 'revisions'
status: 'STARTED' | 'FAILED' | 'FINISHED'
status: 'STARTED' | 'IN_PROGRESS' | 'FAILED' | 'FINISHED'
}): TransitionStatusUpdatedEvent
}

View File

@@ -29,6 +29,14 @@ export class TransitionStatusUpdatedEventHandler implements DomainEventHandlerIn
}
if (event.payload.status === 'STARTED' && event.payload.transitionType === 'revisions') {
await this.domainEventPublisher.publish(
this.domainEventFactory.createTransitionStatusUpdatedEvent({
userUuid: event.payload.userUuid,
status: 'IN_PROGRESS',
transitionType: 'revisions',
}),
)
const result = await this.transitionRevisionsFromPrimaryToSecondaryDatabaseForUser.execute({
userUuid: event.payload.userUuid,
})

View File

@@ -22,6 +22,7 @@ describe('TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser', () => {
secondaryRevisionRepository,
timer,
logger,
1,
)
beforeEach(() => {
@@ -200,9 +201,6 @@ describe('TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser', () => {
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual(
'Revision 00000000-0000-0000-0000-000000000001 is not identical in primary and secondary database',
)
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).toHaveBeenCalledTimes(1)
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()

View File

@@ -11,6 +11,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
private secondRevisionsRepository: RevisionRepositoryInterface | null,
private timer: TimerInterface,
private logger: Logger,
private pageSize: number,
) {}
async execute(dto: TransitionRevisionsFromPrimaryToSecondaryDatabaseForUserDTO): Promise<Result<void>> {
@@ -83,13 +84,12 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
try {
const totalRevisionsCountForUser = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
let totalRevisionsCountTransitionedToSecondary = 0
const pageSize = 1
const totalPages = Math.ceil(totalRevisionsCountForUser / pageSize)
const totalPages = Math.ceil(totalRevisionsCountForUser / this.pageSize)
for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
const query = {
userUuid: userUuid,
offset: (currentPage - 1) * pageSize,
limit: pageSize,
offset: (currentPage - 1) * this.pageSize,
limit: this.pageSize,
}
const revisions = await this.primaryRevisionsRepository.findByUserUuid(query)
@@ -153,13 +153,12 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
try {
const totalRevisionsCountForUserInPrimary = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
const pageSize = 1
const totalPages = Math.ceil(totalRevisionsCountForUserInPrimary / pageSize)
const totalPages = Math.ceil(totalRevisionsCountForUserInPrimary / this.pageSize)
for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
const query = {
userUuid: userUuid,
offset: (currentPage - 1) * pageSize,
limit: pageSize,
offset: (currentPage - 1) * this.pageSize,
limit: this.pageSize,
}
const revisions = await this.primaryRevisionsRepository.findByUserUuid(query)
@@ -180,7 +179,11 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
}
if (!revision.isIdenticalTo(revisionInSecondary)) {
return Result.fail(`Revision ${revision.id.toString()} is not identical in primary and secondary database`)
return Result.fail(
`Revision ${revision.id.toString()} is not identical in primary and secondary database. Revision in primary database: ${JSON.stringify(
revision,
)}, revision in secondary database: ${JSON.stringify(revisionInSecondary)}`,
)
}
}
}

View File

@@ -3,6 +3,30 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.95.4](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.95.3...@standardnotes/syncing-server@1.95.4) (2023-09-12)
### Bug Fixes
* transition adjustments ([f20a947](https://github.com/standardnotes/syncing-server-js/commit/f20a947f8a555c074d8dc1543c7a8bf61d1d887e))
## [1.95.3](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.95.2...@standardnotes/syncing-server@1.95.3) (2023-09-11)
### Bug Fixes
* debug sync block ([1d751c0](https://github.com/standardnotes/syncing-server-js/commit/1d751c0fbe434f661466dde804a37849f23d9b1b))
## [1.95.2](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.95.1...@standardnotes/syncing-server@1.95.2) (2023-09-11)
### Bug Fixes
* **auth:** remove transitioning upon sign out ([#819](https://github.com/standardnotes/syncing-server-js/issues/819)) ([330bff0](https://github.com/standardnotes/syncing-server-js/commit/330bff0124f5f49c3441304d166ea43c21fea7bc))
## [1.95.1](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.95.0...@standardnotes/syncing-server@1.95.1) (2023-09-11)
### Bug Fixes
* disable running migrations in worker mode of a given service ([a82b9a0](https://github.com/standardnotes/syncing-server-js/commit/a82b9a0c8a023ba8a450ff9e34bcd62f928fcab3))
# [1.95.0](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.94.0...@standardnotes/syncing-server@1.95.0) (2023-09-08)
### Features

View File

@@ -24,7 +24,7 @@ const requestTransition = async (
return
}
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
const env: Env = new Env()
env.load()

View File

@@ -7,7 +7,7 @@ import { Env } from '../src/Bootstrap/Env'
import { DomainEventSubscriberFactoryInterface } from '@standardnotes/domain-events'
import { ContainerConfigLoader } from '../src/Bootstrap/Container'
const container = new ContainerConfigLoader()
const container = new ContainerConfigLoader('worker')
void container.load().then((container) => {
const env: Env = new Env()
env.load()

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/syncing-server",
"version": "1.95.0",
"version": "1.95.4",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -173,6 +173,8 @@ export class ContainerConfigLoader {
private readonly DEFAULT_MAX_ITEMS_LIMIT = 300
private readonly DEFAULT_FILE_UPLOAD_PATH = `${__dirname}/../../uploads`
constructor(private mode: 'server' | 'worker' = 'server') {}
async load(configuration?: {
controllerConatiner?: ControllerContainerInterface
directCallDomainEventPublisher?: DirectCallDomainEventPublisher
@@ -211,7 +213,7 @@ export class ContainerConfigLoader {
}
container.bind<winston.Logger>(TYPES.Sync_Logger).toConstantValue(logger)
const appDataSource = new AppDataSource(env)
const appDataSource = new AppDataSource({ env, runMigrations: this.mode === 'server' })
await appDataSource.initialize()
logger.debug('Database initialized')
@@ -831,6 +833,7 @@ export class ContainerConfigLoader {
isSecondaryDatabaseEnabled ? container.get<ItemRepositoryInterface>(TYPES.Sync_MongoDBItemRepository) : null,
container.get<TimerInterface>(TYPES.Sync_Timer),
container.get<Logger>(TYPES.Sync_Logger),
env.get('MIGRATION_BATCH_SIZE', true) ? +env.get('MIGRATION_BATCH_SIZE', true) : 100,
),
)
container

View File

@@ -15,7 +15,12 @@ export class AppDataSource {
private _dataSource: DataSource | undefined
private _secondaryDataSource: DataSource | undefined
constructor(private env: Env) {}
constructor(
private configuration: {
env: Env
runMigrations: boolean
},
) {}
getRepository<Entity extends ObjectLiteral>(target: EntityTarget<Entity>): Repository<Entity> {
if (!this._dataSource) {
@@ -42,20 +47,20 @@ export class AppDataSource {
}
get secondaryDataSource(): DataSource | undefined {
this.env.load()
this.configuration.env.load()
if (this.env.get('SECONDARY_DB_ENABLED', true) !== 'true') {
if (this.configuration.env.get('SECONDARY_DB_ENABLED', true) !== 'true') {
return undefined
}
this._secondaryDataSource = new DataSource({
type: 'mongodb',
host: this.env.get('MONGO_HOST'),
host: this.configuration.env.get('MONGO_HOST'),
authSource: 'admin',
port: parseInt(this.env.get('MONGO_PORT')),
username: this.env.get('MONGO_USERNAME'),
password: this.env.get('MONGO_PASSWORD', true),
database: this.env.get('MONGO_DATABASE'),
port: parseInt(this.configuration.env.get('MONGO_PORT')),
username: this.configuration.env.get('MONGO_USERNAME'),
password: this.configuration.env.get('MONGO_PASSWORD', true),
database: this.configuration.env.get('MONGO_DATABASE'),
entities: [MongoDBItem],
retryWrites: false,
synchronize: true,
@@ -65,14 +70,15 @@ export class AppDataSource {
}
get dataSource(): DataSource {
this.env.load()
this.configuration.env.load()
const isConfiguredForMySQL = this.env.get('DB_TYPE') === 'mysql'
const isConfiguredForMySQL = this.configuration.env.get('DB_TYPE') === 'mysql'
const isConfiguredForHomeServerOrSelfHosting =
this.env.get('MODE', true) === 'home-server' || this.env.get('MODE', true) === 'self-hosted'
this.configuration.env.get('MODE', true) === 'home-server' ||
this.configuration.env.get('MODE', true) === 'self-hosted'
const maxQueryExecutionTime = this.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
? +this.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
const maxQueryExecutionTime = this.configuration.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
? +this.configuration.env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
: 45_000
const migrationsSourceDirectoryName = isConfiguredForMySQL
@@ -92,28 +98,28 @@ export class AppDataSource {
TypeORMMessage,
],
migrations: [`${__dirname}/../../migrations/${migrationsSourceDirectoryName}/*.js`],
migrationsRun: true,
logging: <LoggerOptions>this.env.get('DB_DEBUG_LEVEL', true) ?? 'info',
migrationsRun: this.configuration.runMigrations,
logging: <LoggerOptions>this.configuration.env.get('DB_DEBUG_LEVEL', true) ?? 'info',
}
if (isConfiguredForMySQL) {
const inReplicaMode = this.env.get('DB_REPLICA_HOST', true) ? true : false
const inReplicaMode = this.configuration.env.get('DB_REPLICA_HOST', true) ? true : false
const replicationConfig = {
master: {
host: this.env.get('DB_HOST'),
port: parseInt(this.env.get('DB_PORT')),
username: this.env.get('DB_USERNAME'),
password: this.env.get('DB_PASSWORD'),
database: this.env.get('DB_DATABASE'),
host: this.configuration.env.get('DB_HOST'),
port: parseInt(this.configuration.env.get('DB_PORT')),
username: this.configuration.env.get('DB_USERNAME'),
password: this.configuration.env.get('DB_PASSWORD'),
database: this.configuration.env.get('DB_DATABASE'),
},
slaves: [
{
host: this.env.get('DB_REPLICA_HOST', true),
port: parseInt(this.env.get('DB_PORT')),
username: this.env.get('DB_USERNAME'),
password: this.env.get('DB_PASSWORD'),
database: this.env.get('DB_DATABASE'),
host: this.configuration.env.get('DB_REPLICA_HOST', true),
port: parseInt(this.configuration.env.get('DB_PORT')),
username: this.configuration.env.get('DB_USERNAME'),
password: this.configuration.env.get('DB_PASSWORD'),
database: this.configuration.env.get('DB_DATABASE'),
},
],
removeNodeErrorCount: 10,
@@ -127,11 +133,11 @@ export class AppDataSource {
supportBigNumbers: true,
bigNumberStrings: false,
replication: inReplicaMode ? replicationConfig : undefined,
host: inReplicaMode ? undefined : this.env.get('DB_HOST'),
port: inReplicaMode ? undefined : parseInt(this.env.get('DB_PORT')),
username: inReplicaMode ? undefined : this.env.get('DB_USERNAME'),
password: inReplicaMode ? undefined : this.env.get('DB_PASSWORD'),
database: inReplicaMode ? undefined : this.env.get('DB_DATABASE'),
host: inReplicaMode ? undefined : this.configuration.env.get('DB_HOST'),
port: inReplicaMode ? undefined : parseInt(this.configuration.env.get('DB_PORT')),
username: inReplicaMode ? undefined : this.configuration.env.get('DB_USERNAME'),
password: inReplicaMode ? undefined : this.configuration.env.get('DB_PASSWORD'),
database: inReplicaMode ? undefined : this.configuration.env.get('DB_DATABASE'),
}
this._dataSource = new DataSource(mySQLDataSourceOptions)
@@ -139,7 +145,7 @@ export class AppDataSource {
const sqliteDataSourceOptions: SqliteConnectionOptions = {
...commonDataSourceOptions,
type: 'sqlite',
database: this.env.get('DB_SQLITE_DATABASE_PATH'),
database: this.configuration.env.get('DB_SQLITE_DATABASE_PATH'),
enableWAL: true,
busyErrorRetry: 2000,
}

View File

@@ -4,4 +4,4 @@ import { Env } from './Env'
const env: Env = new Env()
env.load()
export const MigrationsDataSource = new AppDataSource(env).dataSource
export const MigrationsDataSource = new AppDataSource({ env, runMigrations: true }).dataSource

View File

@@ -52,7 +52,7 @@ export interface DomainEventFactoryInterface {
createTransitionStatusUpdatedEvent(dto: {
userUuid: string
transitionType: 'items' | 'revisions'
status: 'STARTED' | 'FAILED' | 'FINISHED'
status: 'STARTED' | 'IN_PROGRESS' | 'FAILED' | 'FINISHED'
}): TransitionStatusUpdatedEvent
createEmailRequestedEvent(dto: {
userEmail: string

View File

@@ -17,6 +17,14 @@ export class TransitionStatusUpdatedEventHandler implements DomainEventHandlerIn
async handle(event: TransitionStatusUpdatedEvent): Promise<void> {
if (event.payload.status === 'STARTED' && event.payload.transitionType === 'items') {
await this.domainEventPublisher.publish(
this.domainEventFactory.createTransitionStatusUpdatedEvent({
userUuid: event.payload.userUuid,
status: 'IN_PROGRESS',
transitionType: 'items',
}),
)
const result = await this.transitionItemsFromPrimaryToSecondaryDatabaseForUser.execute({
userUuid: event.payload.userUuid,
})

View File

@@ -22,6 +22,7 @@ describe('TransitionItemsFromPrimaryToSecondaryDatabaseForUser', () => {
secondaryItemRepository,
timer,
logger,
1,
)
beforeEach(() => {
@@ -205,9 +206,6 @@ describe('TransitionItemsFromPrimaryToSecondaryDatabaseForUser', () => {
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual(
'Item 00000000-0000-0000-0000-000000000001 is not identical in primary and secondary database',
)
expect((secondaryItemRepository as ItemRepositoryInterface).deleteByUserUuid).toHaveBeenCalledTimes(1)
expect(primaryItemRepository.deleteByUserUuid).not.toHaveBeenCalled()

View File

@@ -12,6 +12,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
private secondaryItemRepository: ItemRepositoryInterface | null,
private timer: TimerInterface,
private logger: Logger,
private pageSize: number,
) {}
async execute(dto: TransitionItemsFromPrimaryToSecondaryDatabaseForUserDTO): Promise<Result<void>> {
@@ -92,14 +93,12 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
private async migrateItemsForUser(userUuid: Uuid): Promise<Result<void>> {
try {
const totalItemsCountForUser = await this.primaryItemRepository.countAll({ userUuid: userUuid.value })
const pageSize = 1
const totalPages = totalItemsCountForUser
let currentPage = 1
for (currentPage; currentPage <= totalPages; currentPage++) {
const totalPages = Math.ceil(totalItemsCountForUser / this.pageSize)
for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
const query: ItemQuery = {
userUuid: userUuid.value,
offset: currentPage - 1,
limit: pageSize,
offset: (currentPage - 1) * this.pageSize,
limit: this.pageSize,
}
const items = await this.primaryItemRepository.findAll(query)
@@ -140,14 +139,12 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
)
}
const pageSize = 1
const totalPages = totalItemsCountForUserInPrimary
let currentPage = 1
for (currentPage; currentPage <= totalPages; currentPage++) {
const totalPages = Math.ceil(totalItemsCountForUserInPrimary / this.pageSize)
for (let currentPage = 1; currentPage <= totalPages; currentPage++) {
const query: ItemQuery = {
userUuid: userUuid.value,
offset: currentPage - 1,
limit: pageSize,
offset: (currentPage - 1) * this.pageSize,
limit: this.pageSize,
}
const items = await this.primaryItemRepository.findAll(query)
@@ -159,7 +156,13 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
}
if (!item.isIdenticalTo(itemInSecondary)) {
return Result.fail(`Item ${item.uuid.value} is not identical in primary and secondary database`)
return Result.fail(
`Item ${
item.uuid.value
} is not identical in primary and secondary database. Item in primary database: ${JSON.stringify(
item,
)}, item in secondary database: ${JSON.stringify(itemInSecondary)}`,
)
}
}
}

View File

@@ -36,7 +36,7 @@ export class BaseItemsController extends BaseHttpController {
async sync(request: Request, response: Response): Promise<results.JsonResult> {
if (response.locals.ongoingTransition === true) {
throw new Error('Cannot sync during transition')
throw new Error(`Cannot sync user ${response.locals.user.uuid} during transition`)
}
const itemHashes: ItemHash[] = []