chore(release): publish new version

- @standardnotes/api-gateway@1.89.16
 - @standardnotes/home-server@1.22.44

packages/api-gateway/CHANGELOG.md

@@ -3,6 +3,12 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.89.16](https://github.com/standardnotes/server/compare/@standardnotes/api-gateway@1.89.15...@standardnotes/api-gateway@1.89.16) (2024-01-04)
+
+### Bug Fixes
+
+* **api-gateway:** disable sync request retries ([d2a371b](https://github.com/standardnotes/server/commit/d2a371b92c8b2b7f8921fe57f162e74d4944715d))
+
 ## [1.89.15](https://github.com/standardnotes/server/compare/@standardnotes/api-gateway@1.89.14...@standardnotes/api-gateway@1.89.15) (2024-01-04)
 
 **Note:** Version bump only for package @standardnotes/api-gateway

packages/api-gateway/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/api-gateway",
-  "version": "1.89.15",
+  "version": "1.89.16",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/api-gateway/src/Service/gRPC/GRPCServiceProxy.ts

@@ -134,48 +134,21 @@ export class GRPCServiceProxy implements ServiceProxyInterface {
     request: Request,
     response: Response,
     payload?: Record<string, unknown> | string,
-    retryAttempt?: number,
   ): Promise<void> {
-    try {
-      const result = await this.gRPCSyncingServerServiceProxy.sync(request, response, payload)
+    const result = await this.gRPCSyncingServerServiceProxy.sync(request, response, payload)
 
-      response.status(result.status).send({
-        meta: {
-          auth: {
-            userUuid: response.locals.user?.uuid,
-            roles: response.locals.roles,
-          },
-          server: {
-            filesServerUrl: this.filesServerUrl,
-          },
+    response.status(result.status).send({
+      meta: {
+        auth: {
+          userUuid: response.locals.user?.uuid,
+          roles: response.locals.roles,
         },
-        data: result.data,
-      })
-
-      if (retryAttempt) {
-        this.logger.debug(`Request to Syncing Server succeeded after ${retryAttempt} retries`, {
-          userId: response.locals.user ? response.locals.user.uuid : undefined,
-        })
-      }
-    } catch (error) {
-      const requestDidNotMakeIt =
-        'code' in (error as Record<string, unknown>) && (error as Record<string, unknown>).code === Status.UNAVAILABLE
-
-      const tooManyRetryAttempts = retryAttempt && retryAttempt > 2
-      if (!tooManyRetryAttempts && requestDidNotMakeIt) {
-        await this.timer.sleep(50)
-
-        const nextRetryAttempt = retryAttempt ? retryAttempt + 1 : 1
-
-        this.logger.debug(`Retrying request to Syncing Server for the ${nextRetryAttempt} time`, {
-          userId: response.locals.user ? response.locals.user.uuid : undefined,
-        })
-
-        return this.callSyncingServerGRPC(request, response, payload, nextRetryAttempt)
-      }
-
-      throw error
-    }
+        server: {
+          filesServerUrl: this.filesServerUrl,
+        },
+      },
+      data: result.data,
+    })
   }
 
   async callRevisionsServer(

packages/home-server/CHANGELOG.md

@@ -3,6 +3,30 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.22.44](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.43...@standardnotes/home-server@1.22.44) (2024-01-04)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.22.43](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.42...@standardnotes/home-server@1.22.43) (2024-01-04)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.22.42](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.41...@standardnotes/home-server@1.22.42) (2024-01-04)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.22.41](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.40...@standardnotes/home-server@1.22.41) (2024-01-04)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.22.40](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.39...@standardnotes/home-server@1.22.40) (2024-01-04)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
+## [1.22.39](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.38...@standardnotes/home-server@1.22.39) (2024-01-04)
+
+**Note:** Version bump only for package @standardnotes/home-server
+
 ## [1.22.38](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.37...@standardnotes/home-server@1.22.38) (2024-01-04)
 
 **Note:** Version bump only for package @standardnotes/home-server

packages/home-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/home-server",
-  "version": "1.22.38",
+  "version": "1.22.44",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/syncing-server/CHANGELOG.md

@@ -3,6 +3,36 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [1.132.0](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.131.0...@standardnotes/syncing-server@1.132.0) (2024-01-04)
+
+### Features
+
+* **syncing-server:** send user based metrics to cloudwatch ([0c3bc0c](https://github.com/standardnotes/server/commit/0c3bc0cae654a6783f85e86995f978cc458d8b5c))
+
+# [1.131.0](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.130.3...@standardnotes/syncing-server@1.131.0) (2024-01-04)
+
+### Features
+
+* **syncing-server:** store per user content size utilization and item operations metrics ([4dd2eb9](https://github.com/standardnotes/server/commit/4dd2eb9349eb16006d1ebba99c848b8f6c51baf9))
+
+## [1.130.3](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.130.2...@standardnotes/syncing-server@1.130.3) (2024-01-04)
+
+### Bug Fixes
+
+* **syncing-server:** decrease metric expiration time ([f1aa431](https://github.com/standardnotes/server/commit/f1aa431c223714e56942a32ab75e380baf4f84aa))
+
+## [1.130.2](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.130.1...@standardnotes/syncing-server@1.130.2) (2024-01-04)
+
+### Bug Fixes
+
+* **syncing-server:** amount of minutes to process for metrics ([92bb447](https://github.com/standardnotes/server/commit/92bb447cacd0a40f963c2bfa5e61cb0f451959e6))
+
+## [1.130.1](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.130.0...@standardnotes/syncing-server@1.130.1) (2024-01-04)
+
+### Bug Fixes
+
+* **syncing-server:** skip sending empty metrics ([2c732ff](https://github.com/standardnotes/server/commit/2c732ff713736b845691e5e195f6a99086b6f2d7))
+
 # [1.130.0](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.129.11...@standardnotes/syncing-server@1.130.0) (2024-01-04)
 
 ### Features

packages/syncing-server/bin/statistics.ts

@@ -19,7 +19,7 @@ const sendStatistics = async (
     region: awsRegion,
   })
 
-  const minutesToProcess = 60
+  const minutesToProcess = 30
 
   const metricsToProcess = [Metric.NAMES.ItemCreated, Metric.NAMES.ItemUpdated]
 
@@ -34,6 +34,42 @@ const sendStatistics = async (
         timestamp + Time.MicrosecondsInAMinute,
       )
 
+      if (statistics.sampleCount === 0) {
+        continue
+      }
+
+      await cloudwatchClient.send(
+        new PutMetricDataCommand({
+          Namespace: 'SyncingServer',
+          MetricData: [
+            {
+              MetricName: metricToProcess,
+              Timestamp: dateNMinutesAgo,
+              StatisticValues: {
+                Maximum: statistics.max,
+                Minimum: statistics.min,
+                SampleCount: statistics.sampleCount,
+                Sum: statistics.sum,
+              },
+            },
+          ],
+        }),
+      )
+    }
+  }
+
+  const userMetricsToProcess = [Metric.NAMES.ItemOperation, Metric.NAMES.ContentSizeUtilized]
+  for (const metricToProcess of userMetricsToProcess) {
+    for (let i = 0; i <= minutesToProcess; i++) {
+      const dateNMinutesAgo = timer.getUTCDateNMinutesAgo(minutesToProcess - i)
+      const timestamp = timer.convertDateToMicroseconds(dateNMinutesAgo)
+
+      const statistics = await metricsStore.getUserBasedStatistics(metricToProcess, timestamp)
+
+      if (statistics.sampleCount === 0) {
+        continue
+      }
+
       await cloudwatchClient.send(
         new PutMetricDataCommand({
           Namespace: 'SyncingServer',

packages/syncing-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@standardnotes/syncing-server",
-  "version": "1.130.0",
+  "version": "1.132.0",
   "engines": {
     "node": ">=18.0.0 <21.0.0"
   },

packages/syncing-server/src/Domain/Metrics/Metric.ts

@@ -6,6 +6,8 @@ export class Metric extends ValueObject<MetricProps> {
   static readonly NAMES = {
     ItemCreated: 'ItemCreated',
     ItemUpdated: 'ItemUpdated',
+    ContentSizeUtilized: 'ContentSizeUtilized',
+    ItemOperation: 'ItemOperation',
   }
 
   static create(props: MetricProps): Result<Metric> {

packages/syncing-server/src/Domain/Metrics/MetricsStoreInterface.ts

@@ -1,6 +1,16 @@
 import { Metric } from './Metric'
 
 export interface MetricsStoreInterface {
+  storeUserBasedMetric(metric: Metric, value: number, userUuid: string): Promise<void>
+  getUserBasedStatistics(
+    name: string,
+    timestamp: number,
+  ): Promise<{
+    sum: number
+    max: number
+    min: number
+    sampleCount: number
+  }>
   storeMetric(metric: Metric): Promise<void>
   getStatistics(
     name: string,

packages/syncing-server/src/Domain/UseCase/Syncing/SaveNewItem/SaveNewItem.spec.ts

@@ -27,6 +27,7 @@ describe('SaveNewItem', () => {
 
     metricsStore = {} as jest.Mocked<MetricsStoreInterface>
     metricsStore.storeMetric = jest.fn()
+    metricsStore.storeUserBasedMetric = jest.fn()
 
     item1 = Item.create(
       {

packages/syncing-server/src/Domain/UseCase/Syncing/SaveNewItem/SaveNewItem.ts

@@ -110,6 +110,7 @@ export class SaveNewItem implements UseCaseInterface<Item> {
       return Result.fail(itemOrError.getError())
     }
     const newItem = itemOrError.getValue()
+    newItem.props.contentSize = Buffer.byteLength(JSON.stringify(newItem))
 
     if (dto.itemHash.representsASharedVaultItem()) {
       const sharedVaultAssociationOrError = SharedVaultAssociation.create({
@@ -138,6 +139,15 @@ export class SaveNewItem implements UseCaseInterface<Item> {
 
     await this.itemRepository.insert(newItem)
 
+    await this.metricsStore.storeUserBasedMetric(
+      Metric.create({
+        name: Metric.NAMES.ContentSizeUtilized,
+        timestamp: this.timer.getTimestampInMicroseconds(),
+      }).getValue(),
+      newItem.props.contentSize,
+      userUuid.value,
+    )
+
     await this.metricsStore.storeMetric(
       Metric.create({ name: Metric.NAMES.ItemCreated, timestamp: this.timer.getTimestampInMicroseconds() }).getValue(),
     )

packages/syncing-server/src/Domain/UseCase/Syncing/UpdateExistingItem/UpdateExistingItem.spec.ts

@@ -53,6 +53,7 @@ describe('UpdateExistingItem', () => {
 
     metricsStore = {} as jest.Mocked<MetricsStoreInterface>
     metricsStore.storeMetric = jest.fn()
+    metricsStore.storeUserBasedMetric = jest.fn()
 
     item1 = Item.create(
       {

packages/syncing-server/src/Domain/UseCase/Syncing/UpdateExistingItem/UpdateExistingItem.ts

@@ -180,6 +180,15 @@ export class UpdateExistingItem implements UseCaseInterface<Item> {
       Metric.create({ name: Metric.NAMES.ItemUpdated, timestamp: this.timer.getTimestampInMicroseconds() }).getValue(),
     )
 
+    await this.metricsStore.storeUserBasedMetric(
+      Metric.create({
+        name: Metric.NAMES.ContentSizeUtilized,
+        timestamp: this.timer.getTimestampInMicroseconds(),
+      }).getValue(),
+      dto.existingItem.props.contentSize,
+      userUuid.value,
+    )
+
     /* istanbul ignore next */
     const revisionsFrequency = dto.isFreeUser ? this.freeRevisionFrequency : this.premiumRevisionFrequency
 

packages/syncing-server/src/Infra/Dummy/DummyMetricStore.ts

@@ -2,6 +2,22 @@ import { MetricsStoreInterface } from '../../Domain/Metrics/MetricsStoreInterfac
 import { Metric } from '../../Domain/Metrics/Metric'
 
 export class DummyMetricStore implements MetricsStoreInterface {
+  async getUserBasedStatistics(
+    _name: string,
+    _timestamp: number,
+  ): Promise<{ sum: number; max: number; min: number; sampleCount: number }> {
+    return {
+      sum: 0,
+      max: 0,
+      min: 0,
+      sampleCount: 0,
+    }
+  }
+
+  async storeUserBasedMetric(_metric: Metric, _value: number, _userUuid: string): Promise<void> {
+    // do nothing
+  }
+
   async storeMetric(_metric: Metric): Promise<void> {
     // do nothing
   }

packages/syncing-server/src/Infra/Redis/RedisMetricStore.ts

@@ -6,12 +6,74 @@ import { Metric } from '../../Domain/Metrics/Metric'
 
 export class RedisMetricStore implements MetricsStoreInterface {
   private readonly METRIC_PREFIX = 'metric'
+  private readonly METRIC_PER_USER_PREFIX = 'metric-user'
 
   constructor(
     private redisClient: IORedis.Redis,
     private timer: TimerInterface,
   ) {}
 
+  async storeUserBasedMetric(metric: Metric, value: number, userUuid: string): Promise<void> {
+    const date = this.timer.convertMicrosecondsToDate(metric.props.timestamp)
+    const dateToTheMinuteString = this.timer.convertDateToFormattedString(date, 'YYYY-MM-DD HH:mm')
+    const key = `${this.METRIC_PER_USER_PREFIX}:${userUuid}:${metric.props.name}:${dateToTheMinuteString}`
+
+    const pipeline = this.redisClient.pipeline()
+
+    pipeline.incrbyfloat(key, value)
+    pipeline.incr(`${this.METRIC_PER_USER_PREFIX}:${userUuid}:${Metric.NAMES.ItemOperation}:${dateToTheMinuteString}`)
+
+    const expirationTime = 60 * 60 * 24
+    pipeline.expire(key, expirationTime)
+
+    await pipeline.exec()
+  }
+
+  async getUserBasedStatistics(
+    name: string,
+    timestamp: number,
+  ): Promise<{ sum: number; max: number; min: number; sampleCount: number }> {
+    const date = this.timer.convertMicrosecondsToDate(timestamp)
+    const dateToTheMinuteString = this.timer.convertDateToFormattedString(date, 'YYYY-MM-DD HH:mm')
+
+    const userMetricsKeys = await this.redisClient.keys(
+      `${this.METRIC_PER_USER_PREFIX}:*:${name}:${dateToTheMinuteString}`,
+    )
+
+    let sum = 0
+    let max = 0
+    let min = 0
+    let sampleCount = 0
+
+    const values = await this.redisClient.mget(userMetricsKeys)
+
+    for (const value of values) {
+      if (!value) {
+        continue
+      }
+
+      const valueAsNumber = Number(value)
+
+      sum += valueAsNumber
+      sampleCount++
+
+      if (valueAsNumber > max) {
+        max = valueAsNumber
+      }
+
+      if (valueAsNumber < min) {
+        min = valueAsNumber
+      }
+    }
+
+    return {
+      sum,
+      max,
+      min,
+      sampleCount,
+    }
+  }
+
   async getStatistics(
     name: string,
     from: number,
@@ -64,8 +126,8 @@ export class RedisMetricStore implements MetricsStoreInterface {
 
     pipeline.incr(key)
 
-    const expirationTimeIn24Hours = 60 * 60 * 24
-    pipeline.expire(key, expirationTimeIn24Hours)
+    const expirationTime = 60 * 60 * 6
+    pipeline.expire(key, expirationTime)
 
     await pipeline.exec()
   }