From 87b22ac684732feaf3aef49496b25fbcda5bd888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20S=C3=B3jko?= Date: Tue, 21 Feb 2023 11:45:43 +0100 Subject: [PATCH] fix(syncing-server): creating email backup attachment files --- .../CloudBackupRequestedEventHandler.spec.ts | 10 +++- .../CloudBackupRequestedEventHandler.ts | 9 +++- .../EmailBackupRequestedEventHandler.spec.ts | 10 ++-- .../EmailBackupRequestedEventHandler.ts | 44 ++++++++--------- .../Domain/Item/ItemBackupServiceInterface.ts | 2 +- .../src/Infra/FS/FSItemBackupService.ts | 2 +- .../src/Infra/S3/S3ItemBackupService.ts | 47 +++++++++++++++---- 7 files changed, 84 insertions(+), 40 deletions(-) diff --git a/packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.spec.ts b/packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.spec.ts index d4ca7d60a..6df564c37 100644 --- a/packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.spec.ts +++ b/packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.spec.ts @@ -53,7 +53,7 @@ describe('CloudBackupRequestedEventHandler', () => { } itemBackupService = {} as jest.Mocked - itemBackupService.backup = jest.fn().mockReturnValue('backup-file-name') + itemBackupService.backup = jest.fn().mockReturnValue(['backup-file-name']) logger = {} as jest.Mocked logger.debug = jest.fn() @@ -133,6 +133,14 @@ describe('CloudBackupRequestedEventHandler', () => { expect(expectedError).not.toBeNull() }) + it('should not trigger cloud backup if backup filename is not returned', async () => { + itemBackupService.backup = jest.fn().mockReturnValue([]) + + await createHandler().handle(event) + + expect(extensionsHttpService.triggerCloudBackupOnExtensionsServer).not.toHaveBeenCalled() + }) + it('should trigger cloud backup on extensions server with muted emails', async () => { event.payload.userHasEmailsMuted = true diff --git a/packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.ts b/packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.ts index 15ae13c2f..b331304a2 100644 --- a/packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.ts +++ b/packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.ts @@ -34,14 +34,19 @@ export class CloudBackupRequestedEventHandler implements DomainEventHandlerInter return } - const backupFilename = await this.itemBackupService.backup(items, authParams) + const backupFilenames = await this.itemBackupService.backup(items, authParams) + if (backupFilenames.length === 0) { + this.logger.warn(`No backup files created for user ${event.payload.userUuid}`) + + return + } this.logger.debug(`Sending ${items.length} items to extensions server for user ${event.payload.userUuid}`) await this.extensionsHttpService.triggerCloudBackupOnExtensionsServer({ cloudProvider: event.payload.cloudProvider, authParams, - backupFilename, + backupFilename: backupFilenames[0], forceMute: event.payload.userHasEmailsMuted, muteEmailsSettingUuid: event.payload.muteEmailsSettingUuid, extensionsServerUrl: this.getExtensionsServerUrl(event), diff --git a/packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.spec.ts b/packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.spec.ts index f6d23ee81..984d3997f 100644 --- a/packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.spec.ts +++ b/packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.spec.ts @@ -57,7 +57,7 @@ describe('EmailBackupRequestedEventHandler', () => { } itemBackupService = {} as jest.Mocked - itemBackupService.backup = jest.fn().mockReturnValue('backup-file-name') + itemBackupService.backup = jest.fn().mockReturnValue(['backup-file-name']) domainEventPublisher = {} as jest.Mocked domainEventPublisher.publish = jest.fn() @@ -84,14 +84,14 @@ describe('EmailBackupRequestedEventHandler', () => { it('should inform that multipart backup attachment for email was created', async () => { itemBackupService.backup = jest .fn() - .mockReturnValueOnce('backup-file-name-1') - .mockReturnValueOnce('backup-file-name-2') + .mockReturnValueOnce(['backup-file-name-1']) + .mockReturnValueOnce(['backup-file-name-2', 'backup-file-name-3']) itemTransferCalculator.computeItemUuidBundlesToFetch = jest.fn().mockReturnValue([['1-2-3'], ['2-3-4']]) await createHandler().handle(event) - expect(domainEventPublisher.publish).toHaveBeenCalledTimes(2) - expect(domainEventFactory.createEmailRequestedEvent).toHaveBeenCalledTimes(2) + expect(domainEventPublisher.publish).toHaveBeenCalledTimes(3) + expect(domainEventFactory.createEmailRequestedEvent).toHaveBeenCalledTimes(3) }) it('should not inform that backup attachment for email was created if user key params cannot be obtained', async () => { diff --git a/packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.ts b/packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.ts index 98f561e87..7bedeb2dc 100644 --- a/packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.ts +++ b/packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.ts @@ -59,35 +59,35 @@ export class EmailBackupRequestedEventHandler implements DomainEventHandlerInter sortOrder: 'ASC', }) - const backupFileName = await this.itemBackupService.backup(items, authParams) + const backupFileNames = await this.itemBackupService.backup(items, authParams, this.emailAttachmentMaxByteSize) - this.logger.debug(`Data backed up into: ${backupFileName}`) - - if (backupFileName.length === 0) { + if (backupFileNames.length === 0) { this.logger.error(`Could not create a backup file for user ${event.payload.userUuid}`) return } const dateOnly = new Date().toISOString().substring(0, 10) - await this.domainEventPublisher.publish( - this.domainEventFactory.createEmailRequestedEvent({ - body: getBody(authParams.identifier as string), - level: EmailLevel.LEVELS.System, - messageIdentifier: 'DATA_BACKUP', - subject: getSubject(bundleIndex++, itemUuidBundles.length, dateOnly), - userEmail: authParams.identifier as string, - sender: 'backups@standardnotes.org', - attachments: [ - { - fileName: backupFileName, - filePath: this.s3BackupBucketName, - attachmentFileName: `SN-Data-${dateOnly}.txt`, - attachmentContentType: 'application/json', - }, - ], - }), - ) + for (const backupFileName of backupFileNames) { + await this.domainEventPublisher.publish( + this.domainEventFactory.createEmailRequestedEvent({ + body: getBody(authParams.identifier as string), + level: EmailLevel.LEVELS.System, + messageIdentifier: 'DATA_BACKUP', + subject: getSubject(bundleIndex++, itemUuidBundles.length, dateOnly), + userEmail: authParams.identifier as string, + sender: 'backups@standardnotes.org', + attachments: [ + { + fileName: backupFileName, + filePath: this.s3BackupBucketName, + attachmentFileName: `SN-Data-${dateOnly}.txt`, + attachmentContentType: 'application/json', + }, + ], + }), + ) + } } } } diff --git a/packages/syncing-server/src/Domain/Item/ItemBackupServiceInterface.ts b/packages/syncing-server/src/Domain/Item/ItemBackupServiceInterface.ts index 8604fe5d5..0250b8078 100644 --- a/packages/syncing-server/src/Domain/Item/ItemBackupServiceInterface.ts +++ b/packages/syncing-server/src/Domain/Item/ItemBackupServiceInterface.ts @@ -2,6 +2,6 @@ import { KeyParamsData } from '@standardnotes/responses' import { Item } from './Item' export interface ItemBackupServiceInterface { - backup(items: Array, authParams: KeyParamsData): Promise + backup(items: Array, authParams: KeyParamsData, contentSizeLimit?: number): Promise dump(item: Item): Promise } diff --git a/packages/syncing-server/src/Infra/FS/FSItemBackupService.ts b/packages/syncing-server/src/Infra/FS/FSItemBackupService.ts index 28fc01432..451f4f77e 100644 --- a/packages/syncing-server/src/Infra/FS/FSItemBackupService.ts +++ b/packages/syncing-server/src/Infra/FS/FSItemBackupService.ts @@ -16,7 +16,7 @@ export class FSItemBackupService implements ItemBackupServiceInterface { private logger: Logger, ) {} - async backup(_items: Item[], _authParams: KeyParamsData): Promise { + async backup(_items: Item[], _authParams: KeyParamsData, _contentSizeLimit?: number): Promise { throw new Error('Method not implemented.') } diff --git a/packages/syncing-server/src/Infra/S3/S3ItemBackupService.ts b/packages/syncing-server/src/Infra/S3/S3ItemBackupService.ts index 208559d59..3c8ea686a 100644 --- a/packages/syncing-server/src/Infra/S3/S3ItemBackupService.ts +++ b/packages/syncing-server/src/Infra/S3/S3ItemBackupService.ts @@ -37,21 +37,52 @@ export class S3ItemBackupService implements ItemBackupServiceInterface { return s3Key } - async backup(items: Item[], authParams: KeyParamsData): Promise { + async backup(items: Item[], authParams: KeyParamsData, contentSizeLimit?: number): Promise { if (!this.s3BackupBucketName || this.s3Client === undefined) { this.logger.warn('S3 backup not configured') - return '' + return [] } + const fileNames = [] + let itemProjections: Array = [] + let contentSizeCounter = 0 + for (const item of items) { + const itemProjection = await this.itemProjector.projectFull(item) + + if (contentSizeLimit === undefined) { + itemProjections.push(itemProjection) + + continue + } + + const itemContentSize = Buffer.byteLength(JSON.stringify(itemProjection)) + + if (contentSizeCounter + itemContentSize <= contentSizeLimit) { + itemProjections.push(itemProjection) + + contentSizeCounter += itemContentSize + } else { + const backupFileName = await this.createBackupFile(itemProjections, authParams) + fileNames.push(backupFileName) + + itemProjections = [itemProjection] + contentSizeCounter = itemContentSize + } + } + + if (itemProjections.length > 0) { + const backupFileName = await this.createBackupFile(itemProjections, authParams) + fileNames.push(backupFileName) + } + + return fileNames + } + + private async createBackupFile(itemProjections: ItemProjection[], authParams: KeyParamsData): Promise { const fileName = uuid.v4() - const itemProjections = [] - for (const item of items) { - itemProjections.push(await this.itemProjector.projectFull(item)) - } - - await this.s3Client.send( + await (this.s3Client as S3Client).send( new PutObjectCommand({ Bucket: this.s3BackupBucketName, Key: fileName,