fix(syncing-server): creating email backup attachment files

packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.spec.ts

@@ -53,7 +53,7 @@ describe('CloudBackupRequestedEventHandler', () => {
     }
 
     itemBackupService = {} as jest.Mocked<ItemBackupServiceInterface>
-    itemBackupService.backup = jest.fn().mockReturnValue('backup-file-name')
+    itemBackupService.backup = jest.fn().mockReturnValue(['backup-file-name'])
 
     logger = {} as jest.Mocked<Logger>
     logger.debug = jest.fn()
@@ -133,6 +133,14 @@ describe('CloudBackupRequestedEventHandler', () => {
     expect(expectedError).not.toBeNull()
   })
 
+  it('should not trigger cloud backup if backup filename is not returned', async () => {
+    itemBackupService.backup = jest.fn().mockReturnValue([])
+
+    await createHandler().handle(event)
+
+    expect(extensionsHttpService.triggerCloudBackupOnExtensionsServer).not.toHaveBeenCalled()
+  })
+
   it('should trigger cloud backup on extensions server with muted emails', async () => {
     event.payload.userHasEmailsMuted = true
 

packages/syncing-server/src/Domain/Handler/CloudBackupRequestedEventHandler.ts

@@ -34,14 +34,19 @@ export class CloudBackupRequestedEventHandler implements DomainEventHandlerInter
       return
     }
 
-    const backupFilename = await this.itemBackupService.backup(items, authParams)
+    const backupFilenames = await this.itemBackupService.backup(items, authParams)
+    if (backupFilenames.length === 0) {
+      this.logger.warn(`No backup files created for user ${event.payload.userUuid}`)
+
+      return
+    }
 
     this.logger.debug(`Sending ${items.length} items to extensions server for user ${event.payload.userUuid}`)
 
     await this.extensionsHttpService.triggerCloudBackupOnExtensionsServer({
       cloudProvider: event.payload.cloudProvider,
       authParams,
-      backupFilename,
+      backupFilename: backupFilenames[0],
       forceMute: event.payload.userHasEmailsMuted,
       muteEmailsSettingUuid: event.payload.muteEmailsSettingUuid,
       extensionsServerUrl: this.getExtensionsServerUrl(event),

packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.spec.ts

@@ -57,7 +57,7 @@ describe('EmailBackupRequestedEventHandler', () => {
     }
 
     itemBackupService = {} as jest.Mocked<ItemBackupServiceInterface>
-    itemBackupService.backup = jest.fn().mockReturnValue('backup-file-name')
+    itemBackupService.backup = jest.fn().mockReturnValue(['backup-file-name'])
 
     domainEventPublisher = {} as jest.Mocked<DomainEventPublisherInterface>
     domainEventPublisher.publish = jest.fn()
@@ -84,14 +84,14 @@ describe('EmailBackupRequestedEventHandler', () => {
   it('should inform that multipart backup attachment for email was created', async () => {
     itemBackupService.backup = jest
       .fn()
-      .mockReturnValueOnce('backup-file-name-1')
-      .mockReturnValueOnce('backup-file-name-2')
+      .mockReturnValueOnce(['backup-file-name-1'])
+      .mockReturnValueOnce(['backup-file-name-2', 'backup-file-name-3'])
     itemTransferCalculator.computeItemUuidBundlesToFetch = jest.fn().mockReturnValue([['1-2-3'], ['2-3-4']])
 
     await createHandler().handle(event)
 
-    expect(domainEventPublisher.publish).toHaveBeenCalledTimes(2)
-    expect(domainEventFactory.createEmailRequestedEvent).toHaveBeenCalledTimes(2)
+    expect(domainEventPublisher.publish).toHaveBeenCalledTimes(3)
+    expect(domainEventFactory.createEmailRequestedEvent).toHaveBeenCalledTimes(3)
   })
 
   it('should not inform that backup attachment for email was created if user key params cannot be obtained', async () => {

packages/syncing-server/src/Domain/Handler/EmailBackupRequestedEventHandler.ts

@@ -59,35 +59,35 @@ export class EmailBackupRequestedEventHandler implements DomainEventHandlerInter
         sortOrder: 'ASC',
       })
 
-      const backupFileName = await this.itemBackupService.backup(items, authParams)
+      const backupFileNames = await this.itemBackupService.backup(items, authParams, this.emailAttachmentMaxByteSize)
 
-      this.logger.debug(`Data backed up into: ${backupFileName}`)
-
-      if (backupFileName.length === 0) {
+      if (backupFileNames.length === 0) {
         this.logger.error(`Could not create a backup file for user ${event.payload.userUuid}`)
 
         return
       }
       const dateOnly = new Date().toISOString().substring(0, 10)
 
-      await this.domainEventPublisher.publish(
-        this.domainEventFactory.createEmailRequestedEvent({
-          body: getBody(authParams.identifier as string),
-          level: EmailLevel.LEVELS.System,
-          messageIdentifier: 'DATA_BACKUP',
-          subject: getSubject(bundleIndex++, itemUuidBundles.length, dateOnly),
-          userEmail: authParams.identifier as string,
-          sender: '[email protected]',
-          attachments: [
-            {
-              fileName: backupFileName,
-              filePath: this.s3BackupBucketName,
-              attachmentFileName: `SN-Data-${dateOnly}.txt`,
-              attachmentContentType: 'application/json',
-            },
-          ],
-        }),
-      )
+      for (const backupFileName of backupFileNames) {
+        await this.domainEventPublisher.publish(
+          this.domainEventFactory.createEmailRequestedEvent({
+            body: getBody(authParams.identifier as string),
+            level: EmailLevel.LEVELS.System,
+            messageIdentifier: 'DATA_BACKUP',
+            subject: getSubject(bundleIndex++, itemUuidBundles.length, dateOnly),
+            userEmail: authParams.identifier as string,
+            sender: '[email protected]',
+            attachments: [
+              {
+                fileName: backupFileName,
+                filePath: this.s3BackupBucketName,
+                attachmentFileName: `SN-Data-${dateOnly}.txt`,
+                attachmentContentType: 'application/json',
+              },
+            ],
+          }),
+        )
+      }
     }
   }
 }

packages/syncing-server/src/Domain/Item/ItemBackupServiceInterface.ts

@@ -2,6 +2,6 @@ import { KeyParamsData } from '@standardnotes/responses'
 import { Item } from './Item'
 
 export interface ItemBackupServiceInterface {
-  backup(items: Array<Item>, authParams: KeyParamsData): Promise<string>
+  backup(items: Array<Item>, authParams: KeyParamsData, contentSizeLimit?: number): Promise<string[]>
   dump(item: Item): Promise<string>
 }

packages/syncing-server/src/Infra/FS/FSItemBackupService.ts

@@ -16,7 +16,7 @@ export class FSItemBackupService implements ItemBackupServiceInterface {
     private logger: Logger,
   ) {}
 
-  async backup(_items: Item[], _authParams: KeyParamsData): Promise<string> {
+  async backup(_items: Item[], _authParams: KeyParamsData, _contentSizeLimit?: number): Promise<string[]> {
     throw new Error('Method not implemented.')
   }
 

packages/syncing-server/src/Infra/S3/S3ItemBackupService.ts

@@ -37,21 +37,52 @@ export class S3ItemBackupService implements ItemBackupServiceInterface {
     return s3Key
   }
 
-  async backup(items: Item[], authParams: KeyParamsData): Promise<string> {
+  async backup(items: Item[], authParams: KeyParamsData, contentSizeLimit?: number): Promise<string[]> {
     if (!this.s3BackupBucketName || this.s3Client === undefined) {
       this.logger.warn('S3 backup not configured')
 
-      return ''
+      return []
     }
 
+    const fileNames = []
+    let itemProjections: Array<ItemProjection> = []
+    let contentSizeCounter = 0
+    for (const item of items) {
+      const itemProjection = await this.itemProjector.projectFull(item)
+
+      if (contentSizeLimit === undefined) {
+        itemProjections.push(itemProjection)
+
+        continue
+      }
+
+      const itemContentSize = Buffer.byteLength(JSON.stringify(itemProjection))
+
+      if (contentSizeCounter + itemContentSize <= contentSizeLimit) {
+        itemProjections.push(itemProjection)
+
+        contentSizeCounter += itemContentSize
+      } else {
+        const backupFileName = await this.createBackupFile(itemProjections, authParams)
+        fileNames.push(backupFileName)
+
+        itemProjections = [itemProjection]
+        contentSizeCounter = itemContentSize
+      }
+    }
+
+    if (itemProjections.length > 0) {
+      const backupFileName = await this.createBackupFile(itemProjections, authParams)
+      fileNames.push(backupFileName)
+    }
+
+    return fileNames
+  }
+
+  private async createBackupFile(itemProjections: ItemProjection[], authParams: KeyParamsData): Promise<string> {
     const fileName = uuid.v4()
 
-    const itemProjections = []
-    for (const item of items) {
-      itemProjections.push(await this.itemProjector.projectFull(item))
-    }
-
-    await this.s3Client.send(
+    await (this.s3Client as S3Client).send(
       new PutObjectCommand({
         Bucket: this.s3BackupBucketName,
         Key: fileName,