Compare commits

...

8 Commits

17 changed files with 193 additions and 62 deletions

View File

@@ -3,6 +3,24 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.141.13](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.141.12...@standardnotes/auth-server@1.141.13) (2023-09-14)
### Bug Fixes
* **auth:** set ttl for started and not picked up transitions to 10h ([fe8ca82](https://github.com/standardnotes/server/commit/fe8ca828fb37306e0e5056627e67366885e86861))
## [1.141.12](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.141.11...@standardnotes/auth-server@1.141.12) (2023-09-13)
### Bug Fixes
* **auth:** remove re-triggering revisions transition ([5984e4c](https://github.com/standardnotes/server/commit/5984e4c3e7e550e5ed53805bde1e6dabcbe54da8))
## [1.141.11](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.141.10...@standardnotes/auth-server@1.141.11) (2023-09-13)
### Bug Fixes
* **auth:** passing transition timestamp ([c164bde](https://github.com/standardnotes/server/commit/c164bde847b5974e74fd439f0d439526ad439443))
## [1.141.10](https://github.com/standardnotes/server/compare/@standardnotes/auth-server@1.141.9...@standardnotes/auth-server@1.141.10) (2023-09-13)
### Bug Fixes

View File

@@ -10,7 +10,7 @@ import { Env } from '../src/Bootstrap/Env'
import { DomainEventPublisherInterface } from '@standardnotes/domain-events'
import { DomainEventFactoryInterface } from '../src/Domain/Event/DomainEventFactoryInterface'
import { UserRepositoryInterface } from '../src/Domain/User/UserRepositoryInterface'
import { TransitionStatusRepositoryInterface } from '../src/Domain/Transition/TransitionStatusRepositoryInterface'
import { TimerInterface } from '@standardnotes/time'
const inputArgs = process.argv.slice(2)
const startDateString = inputArgs[0]
@@ -18,23 +18,28 @@ const endDateString = inputArgs[1]
const requestTransition = async (
userRepository: UserRepositoryInterface,
transitionStatusRepository: TransitionStatusRepositoryInterface,
logger: Logger,
domainEventFactory: DomainEventFactoryInterface,
domainEventPublisher: DomainEventPublisherInterface,
timer: TimerInterface,
): Promise<void> => {
const startDate = new Date(startDateString)
const endDate = new Date(endDateString)
const users = await userRepository.findAllCreatedBetween(startDate, endDate)
logger.info(`Found ${users.length} users created between ${startDateString} and ${endDateString}`)
const timestamp = timer.getTimestampInMicroseconds()
logger.info(
`[TRANSITION ${timestamp}] Found ${users.length} users created between ${startDateString} and ${endDateString}`,
)
let usersTriggered = 0
for (const user of users) {
const transitionRequestedEvent = domainEventFactory.createTransitionRequestedEvent({
userUuid: user.uuid,
type: 'items',
timestamp,
})
usersTriggered += 1
@@ -43,22 +48,8 @@ const requestTransition = async (
}
logger.info(
`Triggered transition for ${usersTriggered} users created between ${startDateString} and ${endDateString}`,
`[TRANSITION ${timestamp}] Triggered transition for ${usersTriggered} users created between ${startDateString} and ${endDateString}`,
)
const revisionStatuses = await transitionStatusRepository.getStatuses('revisions')
const failedStatuses = revisionStatuses.filter((status) => status.status === 'FAILED')
logger.info(`Found ${failedStatuses.length} failed revision transitions`)
for (const status of failedStatuses) {
const transitionRequestedEvent = domainEventFactory.createTransitionRequestedEvent({
userUuid: status.userUuid,
type: 'revisions',
})
await domainEventPublisher.publish(transitionRequestedEvent)
}
}
const container = new ContainerConfigLoader('worker')
@@ -75,13 +66,9 @@ void container.load().then((container) => {
const userRepository: UserRepositoryInterface = container.get(TYPES.Auth_UserRepository)
const domainEventFactory: DomainEventFactoryInterface = container.get(TYPES.Auth_DomainEventFactory)
const domainEventPublisher: DomainEventPublisherInterface = container.get(TYPES.Auth_DomainEventPublisher)
const transitionStatusRepository: TransitionStatusRepositoryInterface = container.get(
TYPES.Auth_TransitionStatusRepository,
)
const timer = container.get<TimerInterface>(TYPES.Auth_Timer)
Promise.resolve(
requestTransition(userRepository, transitionStatusRepository, logger, domainEventFactory, domainEventPublisher),
)
Promise.resolve(requestTransition(userRepository, logger, domainEventFactory, domainEventPublisher, timer))
.then(() => {
logger.info(`Finished transition request for users created between ${startDateString} and ${endDateString}`)

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/auth-server",
"version": "1.141.10",
"version": "1.141.13",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -33,7 +33,11 @@ import { DomainEventFactoryInterface } from './DomainEventFactoryInterface'
export class DomainEventFactory implements DomainEventFactoryInterface {
constructor(@inject(TYPES.Auth_Timer) private timer: TimerInterface) {}
createTransitionRequestedEvent(dto: { userUuid: string; type: 'items' | 'revisions' }): TransitionRequestedEvent {
createTransitionRequestedEvent(dto: {
userUuid: string
type: 'items' | 'revisions'
timestamp: number
}): TransitionRequestedEvent {
return {
type: 'TRANSITION_REQUESTED',
createdAt: this.timer.getUTCDate(),
@@ -44,10 +48,7 @@ export class DomainEventFactory implements DomainEventFactoryInterface {
},
origin: DomainEventService.Auth,
},
payload: {
timestamp: this.timer.getTimestampInMicroseconds(),
...dto,
},
payload: dto,
}
}

View File

@@ -90,5 +90,9 @@ export interface DomainEventFactoryInterface {
}): StatisticPersistenceRequestedEvent
createSessionCreatedEvent(dto: { userUuid: string }): SessionCreatedEvent
createSessionRefreshedEvent(dto: { userUuid: string }): SessionRefreshedEvent
createTransitionRequestedEvent(dto: { userUuid: string; type: 'items' | 'revisions' }): TransitionRequestedEvent
createTransitionRequestedEvent(dto: {
userUuid: string
type: 'items' | 'revisions'
timestamp: number
}): TransitionRequestedEvent
}

View File

@@ -30,23 +30,29 @@ export class UpdateTransitionStatus implements UseCaseInterface<void> {
await this.roleService.addRoleToUser(userUuid, RoleName.create(RoleName.NAMES.TransitionUser).getValue())
}
const itemStatuses = await this.transitionStatusRepository.getStatuses('items')
const itemsStartedStatusesCount = itemStatuses.filter((status) => status.status === 'STARTED').length
const itemsInProgressStatusesCount = itemStatuses.filter((status) => status.status === 'IN_PROGRESS').length
const itemsFailedStatusesCount = itemStatuses.filter((status) => status.status === 'FAILED').length
if (dto.transitionType === 'items') {
const itemStatuses = await this.transitionStatusRepository.getStatuses('items')
const itemsStartedStatusesCount = itemStatuses.filter((status) => status.status === 'STARTED').length
const itemsInProgressStatusesCount = itemStatuses.filter((status) => status.status === 'IN_PROGRESS').length
const itemsFailedStatusesCount = itemStatuses.filter((status) => status.status === 'FAILED').length
this.logger.info(
`[TRANSITION ${dto.transitionTimestamp}] Items transition statuses: ${itemsStartedStatusesCount} started, ${itemsInProgressStatusesCount} in progress, ${itemsFailedStatusesCount} failed`,
)
this.logger.info(
`[TRANSITION ${dto.transitionTimestamp}] Items transition statuses: ${itemsStartedStatusesCount} started, ${itemsInProgressStatusesCount} in progress, ${itemsFailedStatusesCount} failed`,
)
}
const revisionStatuses = await this.transitionStatusRepository.getStatuses('revisions')
const revisionsStartedStatusesCount = revisionStatuses.filter((status) => status.status === 'STARTED').length
const revisionsInProgressStatusesCount = revisionStatuses.filter((status) => status.status === 'IN_PROGRESS').length
const revisionsFailedStatusesCount = revisionStatuses.filter((status) => status.status === 'FAILED').length
if (dto.transitionType === 'revisions') {
const revisionStatuses = await this.transitionStatusRepository.getStatuses('revisions')
const revisionsStartedStatusesCount = revisionStatuses.filter((status) => status.status === 'STARTED').length
const revisionsInProgressStatusesCount = revisionStatuses.filter(
(status) => status.status === 'IN_PROGRESS',
).length
const revisionsFailedStatusesCount = revisionStatuses.filter((status) => status.status === 'FAILED').length
this.logger.info(
`[TRANSITION ${dto.transitionTimestamp}] Revisions transition statuses: ${revisionsStartedStatusesCount} started, ${revisionsInProgressStatusesCount} in progress, ${revisionsFailedStatusesCount} failed`,
)
this.logger.info(
`[TRANSITION ${dto.transitionTimestamp}] Revisions transition statuses: ${revisionsStartedStatusesCount} started, ${revisionsInProgressStatusesCount} in progress, ${revisionsFailedStatusesCount} failed`,
)
}
return Result.ok()
}

View File

@@ -28,7 +28,9 @@ export class RedisTransitionStatusRepository implements TransitionStatusReposito
status: 'STARTED' | 'IN_PROGRESS' | 'FAILED',
): Promise<void> {
if (status === 'IN_PROGRESS') {
await this.redisClient.setex(`${this.PREFIX}:${transitionType}:${userUuid}`, 7200, status)
await this.redisClient.setex(`${this.PREFIX}:${transitionType}:${userUuid}`, 7_200, status)
} else if (status === 'STARTED') {
await this.redisClient.setex(`${this.PREFIX}:${transitionType}:${userUuid}`, 36_000, status)
} else {
await this.redisClient.set(`${this.PREFIX}:${transitionType}:${userUuid}`, status)
}

View File

@@ -3,6 +3,18 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.15.54](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.53...@standardnotes/home-server@1.15.54) (2023-09-14)
**Note:** Version bump only for package @standardnotes/home-server
## [1.15.53](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.52...@standardnotes/home-server@1.15.53) (2023-09-13)
**Note:** Version bump only for package @standardnotes/home-server
## [1.15.52](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.51...@standardnotes/home-server@1.15.52) (2023-09-13)
**Note:** Version bump only for package @standardnotes/home-server
## [1.15.51](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.15.50...@standardnotes/home-server@1.15.51) (2023-09-13)
**Note:** Version bump only for package @standardnotes/home-server

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/home-server",
"version": "1.15.51",
"version": "1.15.54",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -3,6 +3,12 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.33.21](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.33.20...@standardnotes/revisions-server@1.33.21) (2023-09-14)
### Bug Fixes
* skip already updated items and revisions in integrity check ([#825](https://github.com/standardnotes/server/issues/825)) ([03a4a3f](https://github.com/standardnotes/server/commit/03a4a3f2abc0b4e09942ba39dbd227524068dfb6))
## [1.33.20](https://github.com/standardnotes/server/compare/@standardnotes/revisions-server@1.33.19...@standardnotes/revisions-server@1.33.20) (2023-09-13)
### Bug Fixes

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/revisions-server",
"version": "1.33.20",
"version": "1.33.21",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -87,6 +87,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
const integrityCheckResult = await this.checkIntegrityBetweenPrimaryAndSecondaryDatabase(
userUuid,
newRevisionsInSecondaryCount,
updatedRevisionsInSecondary,
)
if (integrityCheckResult.isFailed()) {
if (newRevisionsInSecondaryCount === 0 && updatedRevisionsInSecondaryCount === 0) {
@@ -144,6 +145,10 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
(updatedRevision) => updatedRevision.id.toString() === revision.id.toString(),
)
) {
this.logger.info(
`Skipping saving revision ${revision.id.toString()} as it was updated in secondary database`,
)
continue
}
@@ -285,6 +290,7 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
private async checkIntegrityBetweenPrimaryAndSecondaryDatabase(
userUuid: Uuid,
newRevisionsInSecondaryCount: number,
updatedRevisionsInSecondary: Revision[],
): Promise<Result<boolean>> {
try {
const totalRevisionsCountForUserInPrimary = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
@@ -314,6 +320,17 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
return Result.fail(`Revision ${revision.id.toString()} not found in secondary database`)
}
if (
updatedRevisionsInSecondary.find(
(updatedRevision) => updatedRevision.id.toString() === revision.id.toString(),
)
) {
this.logger.info(
`Skipping integrity check for revision ${revision.id.toString()} as it was updated in secondary database`,
)
continue
}
if (!revision.isIdenticalTo(revisionInSecondary)) {
return Result.fail(
`Revision ${revision.id.toString()} is not identical in primary and secondary database. Revision in primary database: ${JSON.stringify(

View File

@@ -3,6 +3,13 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.95.18](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.95.17...@standardnotes/syncing-server@1.95.18) (2023-09-14)
### Bug Fixes
* skip already updated items and revisions in integrity check ([#825](https://github.com/standardnotes/syncing-server-js/issues/825)) ([03a4a3f](https://github.com/standardnotes/syncing-server-js/commit/03a4a3f2abc0b4e09942ba39dbd227524068dfb6))
* **syncing-server:** updating with missing creation date ([#824](https://github.com/standardnotes/syncing-server-js/issues/824)) ([3a8607d](https://github.com/standardnotes/syncing-server-js/commit/3a8607d1465cabedad68b84c753e407342e60d20))
## [1.95.17](https://github.com/standardnotes/syncing-server-js/compare/@standardnotes/syncing-server@1.95.16...@standardnotes/syncing-server@1.95.17) (2023-09-13)
### Bug Fixes

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/syncing-server",
"version": "1.95.17",
"version": "1.95.18",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -298,7 +298,7 @@ describe('UpdateExistingItem', () => {
expect(itemRepository.save).toHaveBeenCalled()
})
it('should return error if created at time is not give in any form', async () => {
it('should fallback to updated at timestamp if created at time is not give in any form', async () => {
const useCase = createUseCase()
const result = await useCase.execute({
@@ -308,13 +308,59 @@ describe('UpdateExistingItem', () => {
...itemHash1.props,
created_at: undefined,
created_at_timestamp: undefined,
updated_at_timestamp: 123,
}).getValue(),
sessionUuid: '00000000-0000-0000-0000-000000000000',
performingUserUuid: '00000000-0000-0000-0000-000000000000',
roleNames: [RoleName.NAMES.CoreUser],
})
expect(result.isFailed()).toBeTruthy()
expect(result.isFailed()).toBeFalsy()
expect(itemRepository.save).toHaveBeenCalled()
})
it('should fallback to updated at date if created at time is not give in any form', async () => {
const useCase = createUseCase()
const result = await useCase.execute({
existingItem: item1,
onGoingRevisionsTransition: false,
itemHash: ItemHash.create({
...itemHash1.props,
created_at: undefined,
created_at_timestamp: undefined,
updated_at_timestamp: undefined,
updated_at: '2020-01-01T00:00:00.000Z',
}).getValue(),
sessionUuid: '00000000-0000-0000-0000-000000000000',
performingUserUuid: '00000000-0000-0000-0000-000000000000',
roleNames: [RoleName.NAMES.CoreUser],
})
expect(result.isFailed()).toBeFalsy()
expect(itemRepository.save).toHaveBeenCalled()
})
it('should fallback to 0 if created at and update at time is not give in any form', async () => {
const useCase = createUseCase()
const result = await useCase.execute({
existingItem: item1,
onGoingRevisionsTransition: false,
itemHash: ItemHash.create({
...itemHash1.props,
created_at: undefined,
created_at_timestamp: undefined,
updated_at_timestamp: undefined,
updated_at: undefined,
}).getValue(),
sessionUuid: '00000000-0000-0000-0000-000000000000',
performingUserUuid: '00000000-0000-0000-0000-000000000000',
roleNames: [RoleName.NAMES.CoreUser],
})
expect(result.isFailed()).toBeFalsy()
expect(itemRepository.save).toHaveBeenCalled()
})
it('should return error if dates could not be created from timestamps', async () => {

View File

@@ -23,6 +23,7 @@ import { SharedVaultOperationOnItem } from '../../../SharedVault/SharedVaultOper
import { AddNotificationForUser } from '../../Messaging/AddNotificationForUser/AddNotificationForUser'
import { RemoveNotificationsForUser } from '../../Messaging/RemoveNotificationsForUser/RemoveNotificationsForUser'
import { ItemRepositoryResolverInterface } from '../../../Item/ItemRepositoryResolverInterface'
import { ItemHash } from '../../../Item/ItemHash'
export class UpdateExistingItem implements UseCaseInterface<Item> {
constructor(
@@ -115,17 +116,7 @@ export class UpdateExistingItem implements UseCaseInterface<Item> {
)
const updatedAtDate = this.timer.convertMicrosecondsToDate(updatedAtTimestamp)
let createdAtTimestamp: number
let createdAtDate: Date
if (dto.itemHash.props.created_at_timestamp) {
createdAtTimestamp = dto.itemHash.props.created_at_timestamp
createdAtDate = this.timer.convertMicrosecondsToDate(createdAtTimestamp)
} else if (dto.itemHash.props.created_at) {
createdAtTimestamp = this.timer.convertStringDateToMicroseconds(dto.itemHash.props.created_at)
createdAtDate = this.timer.convertStringDateToDate(dto.itemHash.props.created_at)
} else {
return Result.fail('Created at timestamp is required.')
}
const { createdAtDate, createdAtTimestamp } = this.determineCreatedAt(dto.itemHash)
const datesOrError = Dates.create(createdAtDate, updatedAtDate)
if (datesOrError.isFailed()) {
@@ -221,6 +212,29 @@ export class UpdateExistingItem implements UseCaseInterface<Item> {
return Result.ok(dto.existingItem)
}
private determineCreatedAt(itemHash: ItemHash): { createdAtDate: Date; createdAtTimestamp: number } {
let createdAtTimestamp: number
let createdAtDate: Date
if (itemHash.props.created_at_timestamp) {
createdAtTimestamp = itemHash.props.created_at_timestamp
createdAtDate = this.timer.convertMicrosecondsToDate(createdAtTimestamp)
} else if (itemHash.props.created_at) {
createdAtTimestamp = this.timer.convertStringDateToMicroseconds(itemHash.props.created_at)
createdAtDate = this.timer.convertStringDateToDate(itemHash.props.created_at)
} else if (itemHash.props.updated_at_timestamp) {
createdAtTimestamp = itemHash.props.updated_at_timestamp
createdAtDate = this.timer.convertMicrosecondsToDate(itemHash.props.updated_at_timestamp)
} else if (itemHash.props.updated_at) {
createdAtTimestamp = this.timer.convertStringDateToMicroseconds(itemHash.props.updated_at)
createdAtDate = this.timer.convertStringDateToDate(itemHash.props.updated_at)
} else {
createdAtTimestamp = 0
createdAtDate = new Date(0)
}
return { createdAtDate, createdAtTimestamp }
}
private async addNotificationsAndPublishEvents(
userUuid: Uuid,
sharedVaultOperation: SharedVaultOperationOnItem | null,

View File

@@ -82,6 +82,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
const integrityCheckResult = await this.checkIntegrityBetweenPrimaryAndSecondaryDatabase(
userUuid,
newItemsInSecondaryCount,
updatedItemsInSecondary,
)
if (integrityCheckResult.isFailed()) {
if (newItemsInSecondaryCount === 0 && updatedItemsInSecondaryCount === 0) {
@@ -209,6 +210,8 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
for (const item of items) {
if (updatedItemsInSecondary.find((updatedItem) => updatedItem.uuid.equals(item.uuid))) {
this.logger.info(`Skipping saving item ${item.uuid.value} as it was updated in secondary database`)
continue
}
await (this.secondaryItemRepository as ItemRepositoryInterface).save(item)
@@ -234,6 +237,7 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
private async checkIntegrityBetweenPrimaryAndSecondaryDatabase(
userUuid: Uuid,
newItemsInSecondaryCount: number,
updatedItemsInSecondary: Item[],
): Promise<Result<boolean>> {
try {
const totalItemsCountForUserInPrimary = await this.primaryItemRepository.countAll({ userUuid: userUuid.value })
@@ -265,6 +269,13 @@ export class TransitionItemsFromPrimaryToSecondaryDatabaseForUser implements Use
return Result.fail(`Item ${item.uuid.value} not found in secondary database`)
}
if (updatedItemsInSecondary.find((updatedItem) => updatedItem.uuid.equals(item.uuid))) {
this.logger.info(
`Skipping integrity check for item ${item.uuid.value} as it was updated in secondary database`,
)
continue
}
if (!item.isIdenticalTo(itemInSecondary)) {
return Result.fail(
`Item ${