Compare commits

..

26 Commits

Author SHA1 Message Date
standardci
b767e1f072 chore(release): publish new version
- @standardnotes/home-server@1.22.49
 - @standardnotes/syncing-server@1.133.3
2024-01-05 12:28:02 +00:00
Karol Sójko
e3cb1faba4 fix(syncing-server): add traffic abuse check in gRPC coms 2024-01-05 13:07:00 +01:00
Karol Sójko
5c5f988055 fix(syncing-server): remove excessive debug logs 2024-01-05 13:07:00 +01:00
standardci
c7d2adf091 chore(release): publish new version
- @standardnotes/home-server@1.22.48
 - @standardnotes/syncing-server@1.133.2
2024-01-05 12:06:26 +00:00
Karol Sójko
a4ad37f309 fix(syncing-server): add debug logs to redis metrics store 2024-01-05 12:45:54 +01:00
Karol Sójko
73c2cc1222 fix(syncing-server): add metadata to transfer breach logs 2024-01-05 12:45:54 +01:00
standardci
9380900aaf chore(release): publish new version
- @standardnotes/home-server@1.22.47
 - @standardnotes/syncing-server@1.133.1
2024-01-05 11:43:04 +00:00
Karol Sójko
02f4d5c717 fix(syncing-server): metadata in logs 2024-01-05 12:22:28 +01:00
Karol Sójko
1f4b26d269 fix(syncing-server): add debug logs for checking traffic abuse 2024-01-05 12:21:24 +01:00
Karol Sójko
e253825da6 fix(syncing-server): error message 2024-01-05 12:09:34 +01:00
standardci
2bddd947ba chore(release): publish new version
- @standardnotes/home-server@1.22.46
 - @standardnotes/syncing-server@1.133.0
2024-01-05 10:50:34 +00:00
Karol Sójko
b7173346d2 feat(syncing-server): add traffic abuse checks (#1014) 2024-01-05 11:30:15 +01:00
standardci
01641975c0 chore(release): publish new version
- @standardnotes/api-gateway@1.89.17
 - @standardnotes/home-server@1.22.45
2024-01-04 18:33:43 +00:00
Karol Sójko
7abd80cdba fix(api-gateway): disable http call retries 2024-01-04 19:13:17 +01:00
standardci
aeb5ea1874 chore(release): publish new version
- @standardnotes/api-gateway@1.89.16
 - @standardnotes/home-server@1.22.44
2024-01-04 18:09:29 +00:00
Karol Sójko
d2a371b92c fix(api-gateway): disable sync request retries 2024-01-04 18:48:55 +01:00
standardci
3ea3b24bb6 chore(release): publish new version
- @standardnotes/home-server@1.22.43
 - @standardnotes/syncing-server@1.132.0
2024-01-04 16:43:22 +00:00
Karol Sójko
0c3bc0cae6 feat(syncing-server): send user based metrics to cloudwatch 2024-01-04 17:22:47 +01:00
standardci
d56410984a chore(release): publish new version
- @standardnotes/home-server@1.22.42
 - @standardnotes/syncing-server@1.131.0
2024-01-04 15:12:28 +00:00
Karol Sójko
4dd2eb9349 feat(syncing-server): store per user content size utilization and item operations metrics 2024-01-04 15:51:10 +01:00
standardci
709aec5eeb chore(release): publish new version
- @standardnotes/home-server@1.22.41
 - @standardnotes/syncing-server@1.130.3
2024-01-04 14:15:26 +00:00
Karol Sójko
f1aa431c22 fix(syncing-server): decrease metric expiration time 2024-01-04 14:54:39 +01:00
standardci
86d0e565ed chore(release): publish new version
- @standardnotes/home-server@1.22.40
 - @standardnotes/syncing-server@1.130.2
2024-01-04 12:53:16 +00:00
Karol Sójko
92bb447cac fix(syncing-server): amount of minutes to process for metrics 2024-01-04 13:32:33 +01:00
standardci
08966e7af7 chore(release): publish new version
- @standardnotes/home-server@1.22.39
 - @standardnotes/syncing-server@1.130.1
2024-01-04 12:24:35 +00:00
Karol Sójko
2c732ff713 fix(syncing-server): skip sending empty metrics 2024-01-04 13:04:03 +01:00
32 changed files with 795 additions and 103 deletions

View File

@@ -3,6 +3,18 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.89.17](https://github.com/standardnotes/server/compare/@standardnotes/api-gateway@1.89.16...@standardnotes/api-gateway@1.89.17) (2024-01-04)
### Bug Fixes
* **api-gateway:** disable http call retries ([7abd80c](https://github.com/standardnotes/server/commit/7abd80cdbaba53840f632d418bd557b35b722699))
## [1.89.16](https://github.com/standardnotes/server/compare/@standardnotes/api-gateway@1.89.15...@standardnotes/api-gateway@1.89.16) (2024-01-04)
### Bug Fixes
* **api-gateway:** disable sync request retries ([d2a371b](https://github.com/standardnotes/server/commit/d2a371b92c8b2b7f8921fe57f162e74d4944715d))
## [1.89.15](https://github.com/standardnotes/server/compare/@standardnotes/api-gateway@1.89.14...@standardnotes/api-gateway@1.89.15) (2024-01-04)
**Note:** Version bump only for package @standardnotes/api-gateway

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/api-gateway",
"version": "1.89.15",
"version": "1.89.17",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -175,7 +175,6 @@ export class HttpServiceProxy implements ServiceProxyInterface {
response: Response,
endpoint: string,
payload?: Record<string, unknown> | string,
retryAttempt?: number,
): Promise<AxiosResponse | undefined> {
try {
const headers: Record<string, string> = {}
@@ -213,33 +212,15 @@ export class HttpServiceProxy implements ServiceProxyInterface {
await this.crossServiceTokenCache.invalidate(userUuid)
}
if (retryAttempt) {
this.logger.debug(`Request to ${serverUrl}/${endpoint} succeeded after ${retryAttempt} retries`)
}
return serviceResponse
} catch (error) {
const requestDidNotMakeIt = this.requestTimedOutOrDidNotReachDestination(error as Record<string, unknown>)
const tooManyRetryAttempts = retryAttempt && retryAttempt > 2
if (!tooManyRetryAttempts && requestDidNotMakeIt) {
await this.timer.sleep(50)
const nextRetryAttempt = retryAttempt ? retryAttempt + 1 : 1
this.logger.debug(`Retrying request to ${serverUrl}/${endpoint} for the ${nextRetryAttempt} time`)
return this.getServerResponse(serverUrl, request, response, endpoint, payload, nextRetryAttempt)
}
let detailedErrorMessage = (error as Error).message
if (error instanceof AxiosError) {
detailedErrorMessage = `Status: ${error.status}, code: ${error.code}, message: ${error.message}`
}
this.logger.error(
tooManyRetryAttempts
? `Request to ${serverUrl}/${endpoint} timed out after ${retryAttempt} retries`
: `Could not pass the request to ${serverUrl}/${endpoint} on underlying service: ${detailedErrorMessage}`,
`Could not pass the request to ${serverUrl}/${endpoint} on underlying service: ${detailedErrorMessage}`,
{
userId: response.locals.user ? response.locals.user.uuid : undefined,
},

View File

@@ -134,48 +134,21 @@ export class GRPCServiceProxy implements ServiceProxyInterface {
request: Request,
response: Response,
payload?: Record<string, unknown> | string,
retryAttempt?: number,
): Promise<void> {
try {
const result = await this.gRPCSyncingServerServiceProxy.sync(request, response, payload)
const result = await this.gRPCSyncingServerServiceProxy.sync(request, response, payload)
response.status(result.status).send({
meta: {
auth: {
userUuid: response.locals.user?.uuid,
roles: response.locals.roles,
},
server: {
filesServerUrl: this.filesServerUrl,
},
response.status(result.status).send({
meta: {
auth: {
userUuid: response.locals.user?.uuid,
roles: response.locals.roles,
},
data: result.data,
})
if (retryAttempt) {
this.logger.debug(`Request to Syncing Server succeeded after ${retryAttempt} retries`, {
userId: response.locals.user ? response.locals.user.uuid : undefined,
})
}
} catch (error) {
const requestDidNotMakeIt =
'code' in (error as Record<string, unknown>) && (error as Record<string, unknown>).code === Status.UNAVAILABLE
const tooManyRetryAttempts = retryAttempt && retryAttempt > 2
if (!tooManyRetryAttempts && requestDidNotMakeIt) {
await this.timer.sleep(50)
const nextRetryAttempt = retryAttempt ? retryAttempt + 1 : 1
this.logger.debug(`Retrying request to Syncing Server for the ${nextRetryAttempt} time`, {
userId: response.locals.user ? response.locals.user.uuid : undefined,
})
return this.callSyncingServerGRPC(request, response, payload, nextRetryAttempt)
}
throw error
}
server: {
filesServerUrl: this.filesServerUrl,
},
},
data: result.data,
})
}
async callRevisionsServer(

View File

@@ -3,6 +3,50 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.22.49](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.48...@standardnotes/home-server@1.22.49) (2024-01-05)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.48](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.47...@standardnotes/home-server@1.22.48) (2024-01-05)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.47](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.46...@standardnotes/home-server@1.22.47) (2024-01-05)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.46](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.45...@standardnotes/home-server@1.22.46) (2024-01-05)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.45](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.44...@standardnotes/home-server@1.22.45) (2024-01-04)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.44](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.43...@standardnotes/home-server@1.22.44) (2024-01-04)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.43](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.42...@standardnotes/home-server@1.22.43) (2024-01-04)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.42](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.41...@standardnotes/home-server@1.22.42) (2024-01-04)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.41](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.40...@standardnotes/home-server@1.22.41) (2024-01-04)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.40](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.39...@standardnotes/home-server@1.22.40) (2024-01-04)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.39](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.38...@standardnotes/home-server@1.22.39) (2024-01-04)
**Note:** Version bump only for package @standardnotes/home-server
## [1.22.38](https://github.com/standardnotes/server/compare/@standardnotes/home-server@1.22.37...@standardnotes/home-server@1.22.38) (2024-01-04)
**Note:** Version bump only for package @standardnotes/home-server

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/home-server",
"version": "1.22.38",
"version": "1.22.49",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -3,6 +3,64 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
## [1.133.3](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.133.2...@standardnotes/syncing-server@1.133.3) (2024-01-05)
### Bug Fixes
* **syncing-server:** add traffic abuse check in gRPC coms ([e3cb1fa](https://github.com/standardnotes/server/commit/e3cb1faba46bbfd8741f6c827daa9438934dd710))
* **syncing-server:** remove excessive debug logs ([5c5f988](https://github.com/standardnotes/server/commit/5c5f9880556f14f5cbe4599ac0d639c970495056))
## [1.133.2](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.133.1...@standardnotes/syncing-server@1.133.2) (2024-01-05)
### Bug Fixes
* **syncing-server:** add debug logs to redis metrics store ([a4ad37f](https://github.com/standardnotes/server/commit/a4ad37f30948ba4292f367240c3dbfca916282ac))
* **syncing-server:** add metadata to transfer breach logs ([73c2cc1](https://github.com/standardnotes/server/commit/73c2cc1222b647e82de3755c7e28d283cd9f872f))
## [1.133.1](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.133.0...@standardnotes/syncing-server@1.133.1) (2024-01-05)
### Bug Fixes
* **syncing-server:** add debug logs for checking traffic abuse ([1f4b26d](https://github.com/standardnotes/server/commit/1f4b26d269a92f5b43455ce3a3cf3d4f15f0d099))
* **syncing-server:** error message ([e253825](https://github.com/standardnotes/server/commit/e253825da69d1be6d7bb4d0360f8c3add73516ef))
* **syncing-server:** metadata in logs ([02f4d5c](https://github.com/standardnotes/server/commit/02f4d5c717cef1014930f11b2792868967812042))
# [1.133.0](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.132.0...@standardnotes/syncing-server@1.133.0) (2024-01-05)
### Features
* **syncing-server:** add traffic abuse checks ([#1014](https://github.com/standardnotes/server/issues/1014)) ([b717334](https://github.com/standardnotes/server/commit/b7173346d2949269b762b023da9ea67b7f327c35))
# [1.132.0](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.131.0...@standardnotes/syncing-server@1.132.0) (2024-01-04)
### Features
* **syncing-server:** send user based metrics to cloudwatch ([0c3bc0c](https://github.com/standardnotes/server/commit/0c3bc0cae654a6783f85e86995f978cc458d8b5c))
# [1.131.0](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.130.3...@standardnotes/syncing-server@1.131.0) (2024-01-04)
### Features
* **syncing-server:** store per user content size utilization and item operations metrics ([4dd2eb9](https://github.com/standardnotes/server/commit/4dd2eb9349eb16006d1ebba99c848b8f6c51baf9))
## [1.130.3](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.130.2...@standardnotes/syncing-server@1.130.3) (2024-01-04)
### Bug Fixes
* **syncing-server:** decrease metric expiration time ([f1aa431](https://github.com/standardnotes/server/commit/f1aa431c223714e56942a32ab75e380baf4f84aa))
## [1.130.2](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.130.1...@standardnotes/syncing-server@1.130.2) (2024-01-04)
### Bug Fixes
* **syncing-server:** amount of minutes to process for metrics ([92bb447](https://github.com/standardnotes/server/commit/92bb447cacd0a40f963c2bfa5e61cb0f451959e6))
## [1.130.1](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.130.0...@standardnotes/syncing-server@1.130.1) (2024-01-04)
### Bug Fixes
* **syncing-server:** skip sending empty metrics ([2c732ff](https://github.com/standardnotes/server/commit/2c732ff713736b845691e5e195f6a99086b6f2d7))
# [1.130.0](https://github.com/standardnotes/server/compare/@standardnotes/syncing-server@1.129.11...@standardnotes/syncing-server@1.130.0) (2024-01-04)
### Features

View File

@@ -29,6 +29,7 @@ import { SyncingServer } from '../src/Infra/gRPC/SyncingServer'
import { SyncItems } from '../src/Domain/UseCase/Syncing/SyncItems/SyncItems'
import { SyncResponseFactoryResolverInterface } from '../src/Domain/Item/SyncResponse/SyncResponseFactoryResolverInterface'
import { SyncResponse20200115 } from '../src/Domain/Item/SyncResponse/SyncResponse20200115'
import { CheckForTrafficAbuse } from '../src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse'
const container = new ContainerConfigLoader()
void container.load().then((container) => {
@@ -114,6 +115,12 @@ void container.load().then((container) => {
container.get<SyncItems>(TYPES.Sync_SyncItems),
container.get<SyncResponseFactoryResolverInterface>(TYPES.Sync_SyncResponseFactoryResolver),
container.get<MapperInterface<SyncResponse20200115, SyncResponse>>(TYPES.Sync_SyncResponseGRPCMapper),
container.get<CheckForTrafficAbuse>(TYPES.Sync_CheckForTrafficAbuse),
container.get<boolean>(TYPES.Sync_STRICT_ABUSE_PROTECTION),
container.get<number>(TYPES.Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES),
container.get<number>(TYPES.Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD),
container.get<number>(TYPES.Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD),
container.get<number>(TYPES.Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES),
container.get<winston.Logger>(TYPES.Sync_Logger),
)

View File

@@ -19,7 +19,7 @@ const sendStatistics = async (
region: awsRegion,
})
const minutesToProcess = 60
const minutesToProcess = 30
const metricsToProcess = [Metric.NAMES.ItemCreated, Metric.NAMES.ItemUpdated]
@@ -28,12 +28,48 @@ const sendStatistics = async (
const dateNMinutesAgo = timer.getUTCDateNMinutesAgo(minutesToProcess - i)
const timestamp = timer.convertDateToMicroseconds(dateNMinutesAgo)
const statistics = await metricsStore.getStatistics(
const statistics = await metricsStore.getMetricsSummary(
metricToProcess,
timestamp,
timestamp + Time.MicrosecondsInAMinute,
)
if (statistics.sampleCount === 0) {
continue
}
await cloudwatchClient.send(
new PutMetricDataCommand({
Namespace: 'SyncingServer',
MetricData: [
{
MetricName: metricToProcess,
Timestamp: dateNMinutesAgo,
StatisticValues: {
Maximum: statistics.max,
Minimum: statistics.min,
SampleCount: statistics.sampleCount,
Sum: statistics.sum,
},
},
],
}),
)
}
}
const userMetricsToProcess = [Metric.NAMES.ItemOperation, Metric.NAMES.ContentSizeUtilized]
for (const metricToProcess of userMetricsToProcess) {
for (let i = 0; i <= minutesToProcess; i++) {
const dateNMinutesAgo = timer.getUTCDateNMinutesAgo(minutesToProcess - i)
const timestamp = timer.convertDateToMicroseconds(dateNMinutesAgo)
const statistics = await metricsStore.getUserBasedMetricsSummary(metricToProcess, timestamp)
if (statistics.sampleCount === 0) {
continue
}
await cloudwatchClient.send(
new PutMetricDataCommand({
Namespace: 'SyncingServer',

View File

@@ -1,6 +1,6 @@
{
"name": "@standardnotes/syncing-server",
"version": "1.130.0",
"version": "1.133.3",
"engines": {
"node": ">=18.0.0 <21.0.0"
},

View File

@@ -166,6 +166,7 @@ import { SendEventToClients } from '../Domain/UseCase/Syncing/SendEventToClients
import { MetricsStoreInterface } from '../Domain/Metrics/MetricsStoreInterface'
import { RedisMetricStore } from '../Infra/Redis/RedisMetricStore'
import { DummyMetricStore } from '../Infra/Dummy/DummyMetricStore'
import { CheckForTrafficAbuse } from '../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse'
export class ContainerConfigLoader {
private readonly DEFAULT_CONTENT_SIZE_TRANSFER_LIMIT = 10_000_000
@@ -472,6 +473,33 @@ export class ContainerConfigLoader {
})
// env vars
container
.bind(TYPES.Sync_STRICT_ABUSE_PROTECTION)
.toConstantValue(env.get('STRICT_ABUSE_PROTECTION', true) === 'true')
container
.bind(TYPES.Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD)
.toConstantValue(
env.get('ITEM_OPERATIONS_ABUSE_THRESHOLD', true) ? +env.get('ITEM_OPERATIONS_ABUSE_THRESHOLD', true) : 500,
)
container
.bind(TYPES.Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES)
.toConstantValue(
env.get('ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', true)
? +env.get('ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', true)
: 5,
)
container
.bind(TYPES.Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD)
.toConstantValue(
env.get('PAYLOAD_SIZE_ABUSE_THRESHOLD', true) ? +env.get('PAYLOAD_SIZE_ABUSE_THRESHOLD', true) : 20_000_000,
)
container
.bind(TYPES.Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES)
.toConstantValue(
env.get('PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', true)
? +env.get('PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', true)
: 5,
)
container.bind(TYPES.Sync_AUTH_JWT_SECRET).toConstantValue(env.get('AUTH_JWT_SECRET'))
container
.bind(TYPES.Sync_REVISIONS_FREQUENCY)
@@ -557,7 +585,11 @@ export class ContainerConfigLoader {
container
.bind<MetricsStoreInterface>(TYPES.Sync_MetricsStore)
.toConstantValue(
new RedisMetricStore(container.get<Redis>(TYPES.Sync_Redis), container.get<TimerInterface>(TYPES.Sync_Timer)),
new RedisMetricStore(
container.get<Redis>(TYPES.Sync_Redis),
container.get<TimerInterface>(TYPES.Sync_Timer),
container.get<Logger>(TYPES.Sync_Logger),
),
)
}
@@ -933,6 +965,15 @@ export class ContainerConfigLoader {
context.container.get(TYPES.Sync_SyncResponseFactory20200115),
)
})
container
.bind<CheckForTrafficAbuse>(TYPES.Sync_CheckForTrafficAbuse)
.toConstantValue(
new CheckForTrafficAbuse(
container.get<MetricsStoreInterface>(TYPES.Sync_MetricsStore),
container.get<TimerInterface>(TYPES.Sync_Timer),
container.get<Logger>(TYPES.Sync_Logger),
),
)
// Handlers
container
@@ -1094,11 +1135,18 @@ export class ContainerConfigLoader {
.bind<BaseItemsController>(TYPES.Sync_BaseItemsController)
.toConstantValue(
new BaseItemsController(
container.get<CheckForTrafficAbuse>(TYPES.Sync_CheckForTrafficAbuse),
container.get<SyncItems>(TYPES.Sync_SyncItems),
container.get<CheckIntegrity>(TYPES.Sync_CheckIntegrity),
container.get<GetItem>(TYPES.Sync_GetItem),
container.get<MapperInterface<Item, ItemHttpRepresentation>>(TYPES.Sync_ItemHttpMapper),
container.get<SyncResponseFactoryResolverInterface>(TYPES.Sync_SyncResponseFactoryResolver),
container.get<Logger>(TYPES.Sync_Logger),
container.get<boolean>(TYPES.Sync_STRICT_ABUSE_PROTECTION),
container.get<number>(TYPES.Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES),
container.get<number>(TYPES.Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD),
container.get<number>(TYPES.Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD),
container.get<number>(TYPES.Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES),
container.get<ControllerContainerInterface>(TYPES.Sync_ControllerContainer),
),
)

View File

@@ -42,6 +42,15 @@ const TYPES = {
Sync_VALET_TOKEN_SECRET: Symbol.for('Sync_VALET_TOKEN_SECRET'),
Sync_VALET_TOKEN_TTL: Symbol.for('Sync_VALET_TOKEN_TTL'),
Sync_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING: Symbol.for('Sync_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING'),
Sync_STRICT_ABUSE_PROTECTION: Symbol.for('Sync_STRICT_ABUSE_PROTECTION'),
Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES: Symbol.for(
'Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES',
),
Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD: Symbol.for('Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD'),
Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD: Symbol.for('Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD'),
Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES: Symbol.for(
'Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES',
),
// use cases
Sync_SyncItems: Symbol.for('Sync_SyncItems'),
Sync_CheckIntegrity: Symbol.for('Sync_CheckIntegrity'),
@@ -85,6 +94,7 @@ const TYPES = {
Sync_TransferSharedVault: Symbol.for('Sync_TransferSharedVault'),
Sync_TransferSharedVaultItems: Symbol.for('Sync_TransferSharedVaultItems'),
Sync_DumpItem: Symbol.for('Sync_DumpItem'),
Sync_CheckForTrafficAbuse: Symbol.for('Sync_CheckForTrafficAbuse'),
// Handlers
Sync_AccountDeletionRequestedEventHandler: Symbol.for('Sync_AccountDeletionRequestedEventHandler'),
Sync_AccountDeletionVerificationRequestedEventHandler: Symbol.for(

View File

@@ -3,7 +3,7 @@ import {
DomainEventPublisherInterface,
EmailBackupRequestedEvent,
} from '@standardnotes/domain-events'
import { EmailLevel } from '@standardnotes/domain-core'
import { EmailLevel, Uuid } from '@standardnotes/domain-core'
import { Logger } from 'winston'
import { DomainEventFactoryInterface } from '../Event/DomainEventFactoryInterface'
import { ItemBackupServiceInterface } from '../Item/ItemBackupServiceInterface'
@@ -32,6 +32,17 @@ export class EmailBackupRequestedEventHandler implements DomainEventHandlerInter
event: EmailBackupRequestedEvent,
itemRepository: ItemRepositoryInterface,
): Promise<void> {
const userUuidOrError = Uuid.create(event.payload.userUuid)
if (userUuidOrError.isFailed()) {
this.logger.error('User uuid is invalid', {
userId: event.payload.userUuid,
codeTag: 'EmailBackupRequestedEventHandler',
})
return
}
const userUuid = userUuidOrError.getValue()
const itemQuery: ItemQuery = {
userUuid: event.payload.userUuid,
sortBy: 'updated_at_timestamp',
@@ -42,6 +53,7 @@ export class EmailBackupRequestedEventHandler implements DomainEventHandlerInter
const itemUuidBundles = await this.itemTransferCalculator.computeItemUuidBundlesToFetch(
itemContentSizeDescriptors,
this.emailAttachmentMaxByteSize,
userUuid,
)
const backupFileNames: string[] = []

View File

@@ -4,13 +4,18 @@ import { Logger } from 'winston'
import { ItemTransferCalculator } from './ItemTransferCalculator'
import { ItemContentSizeDescriptor } from './ItemContentSizeDescriptor'
import { Uuid } from '@standardnotes/domain-core'
describe('ItemTransferCalculator', () => {
let logger: Logger
const createCalculator = () => new ItemTransferCalculator(logger)
let userUuid: Uuid
beforeEach(() => {
userUuid = Uuid.create('00000000-0000-0000-0000-000000000000').getValue()
logger = {} as jest.Mocked<Logger>
logger.warn = jest.fn()
})
@@ -23,7 +28,7 @@ describe('ItemTransferCalculator', () => {
ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000002', 20).getValue(),
]
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 50)
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 50, userUuid)
expect(result).toEqual({
uuids: [
@@ -42,7 +47,7 @@ describe('ItemTransferCalculator', () => {
ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000002', 20).getValue(),
]
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 40)
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 40, userUuid)
expect(result).toEqual({
uuids: ['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'],
@@ -57,7 +62,7 @@ describe('ItemTransferCalculator', () => {
ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000002', null).getValue(),
]
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 50)
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 50, userUuid)
expect(result).toEqual({
uuids: [
@@ -76,7 +81,7 @@ describe('ItemTransferCalculator', () => {
ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000002', 20).getValue(),
]
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 40)
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 40, userUuid)
expect(result).toEqual({
uuids: ['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'],
@@ -93,7 +98,7 @@ describe('ItemTransferCalculator', () => {
ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000002', 20).getValue(),
]
const result = await createCalculator().computeItemUuidBundlesToFetch(itemContentSizeDescriptors, 50)
const result = await createCalculator().computeItemUuidBundlesToFetch(itemContentSizeDescriptors, 50, userUuid)
expect(result).toEqual([
[
@@ -111,7 +116,7 @@ describe('ItemTransferCalculator', () => {
ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000002', 20).getValue(),
]
const result = await createCalculator().computeItemUuidBundlesToFetch(itemContentSizeDescriptors, 40)
const result = await createCalculator().computeItemUuidBundlesToFetch(itemContentSizeDescriptors, 40, userUuid)
expect(result).toEqual([
['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'],
@@ -126,7 +131,7 @@ describe('ItemTransferCalculator', () => {
ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000002', null).getValue(),
]
const result = await createCalculator().computeItemUuidBundlesToFetch(itemContentSizeDescriptors, 50)
const result = await createCalculator().computeItemUuidBundlesToFetch(itemContentSizeDescriptors, 50, userUuid)
expect(result).toEqual([
[
@@ -144,7 +149,7 @@ describe('ItemTransferCalculator', () => {
ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000002', 20).getValue(),
]
const result = await createCalculator().computeItemUuidBundlesToFetch(itemContentSizeDescriptors, 40)
const result = await createCalculator().computeItemUuidBundlesToFetch(itemContentSizeDescriptors, 40, userUuid)
expect(result).toEqual([
['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'],

View File

@@ -2,6 +2,7 @@ import { Logger } from 'winston'
import { ItemTransferCalculatorInterface } from './ItemTransferCalculatorInterface'
import { ItemContentSizeDescriptor } from './ItemContentSizeDescriptor'
import { Uuid } from '@standardnotes/domain-core'
export class ItemTransferCalculator implements ItemTransferCalculatorInterface {
constructor(private logger: Logger) {}
@@ -9,6 +10,7 @@ export class ItemTransferCalculator implements ItemTransferCalculatorInterface {
async computeItemUuidsToFetch(
itemContentSizeDescriptors: ItemContentSizeDescriptor[],
bytesTransferLimit: number,
userUuid: Uuid,
): Promise<{ uuids: Array<string>; transferLimitBreachedBeforeEndOfItems: boolean }> {
const itemUuidsToFetch = []
let totalContentSizeInBytes = 0
@@ -24,6 +26,7 @@ export class ItemTransferCalculator implements ItemTransferCalculatorInterface {
bytesTransferLimit,
itemUuidsToFetch,
itemContentSizeDescriptors,
userUuid,
})
if (transferLimitBreached) {
@@ -41,6 +44,7 @@ export class ItemTransferCalculator implements ItemTransferCalculatorInterface {
async computeItemUuidBundlesToFetch(
itemContentSizeDescriptors: ItemContentSizeDescriptor[],
bytesTransferLimit: number,
userUuid: Uuid,
): Promise<Array<Array<string>>> {
let itemUuidsToFetch = []
let totalContentSizeInBytes = 0
@@ -56,6 +60,7 @@ export class ItemTransferCalculator implements ItemTransferCalculatorInterface {
bytesTransferLimit,
itemUuidsToFetch,
itemContentSizeDescriptors,
userUuid,
})
if (transferLimitBreached) {
@@ -77,15 +82,20 @@ export class ItemTransferCalculator implements ItemTransferCalculatorInterface {
bytesTransferLimit: number
itemUuidsToFetch: Array<string>
itemContentSizeDescriptors: ItemContentSizeDescriptor[]
userUuid: Uuid
}): boolean {
const transferLimitBreached = dto.totalContentSizeInBytes >= dto.bytesTransferLimit
const transferLimitBreachedAtFirstItem =
transferLimitBreached && dto.itemUuidsToFetch.length === 1 && dto.itemContentSizeDescriptors.length > 1
if (transferLimitBreachedAtFirstItem) {
this.logger.warn(
`Item ${dto.itemUuidsToFetch[0]} is breaching the content size transfer limit: ${dto.bytesTransferLimit}`,
)
this.logger.warn('Item is breaching the content size transfer limit at first item in the bundle to fetch.', {
codeTag: 'ItemTransferCalculator',
itemUuid: dto.itemUuidsToFetch[0],
totalContentSizeInBytes: dto.totalContentSizeInBytes,
bytesTransferLimit: dto.bytesTransferLimit,
userId: dto.userUuid.value,
})
}
return transferLimitBreached && !transferLimitBreachedAtFirstItem

View File

@@ -1,12 +1,16 @@
import { Uuid } from '@standardnotes/domain-core'
import { ItemContentSizeDescriptor } from './ItemContentSizeDescriptor'
export interface ItemTransferCalculatorInterface {
computeItemUuidsToFetch(
itemContentSizeDescriptors: ItemContentSizeDescriptor[],
bytesTransferLimit: number,
userUuid: Uuid,
): Promise<{ uuids: Array<string>; transferLimitBreachedBeforeEndOfItems: boolean }>
computeItemUuidBundlesToFetch(
itemContentSizeDescriptors: ItemContentSizeDescriptor[],
bytesTransferLimit: number,
userUuid: Uuid,
): Promise<Array<Array<string>>>
}

View File

@@ -6,6 +6,8 @@ export class Metric extends ValueObject<MetricProps> {
static readonly NAMES = {
ItemCreated: 'ItemCreated',
ItemUpdated: 'ItemUpdated',
ContentSizeUtilized: 'ContentSizeUtilized',
ItemOperation: 'ItemOperation',
}
static create(props: MetricProps): Result<Metric> {

View File

@@ -1,15 +1,17 @@
import { Uuid } from '@standardnotes/domain-core'
import { Metric } from './Metric'
import { MetricsSummary } from './MetricsSummary'
export interface MetricsStoreInterface {
storeMetric(metric: Metric): Promise<void>
getStatistics(
name: string,
from: number,
to: number,
): Promise<{
sum: number
max: number
min: number
sampleCount: number
}>
storeUserBasedMetric(metric: Metric, value: number, userUuid: Uuid): Promise<void>
getUserBasedMetricsSummaryWithinTimeRange(dto: {
metricName: string
userUuid: Uuid
from: Date
to: Date
}): Promise<MetricsSummary>
getUserBasedMetricsSummary(name: string, timestamp: number): Promise<MetricsSummary>
getMetricsSummary(name: string, from: number, to: number): Promise<MetricsSummary>
}

View File

@@ -0,0 +1,6 @@
export interface MetricsSummary {
sum: number
max: number
min: number
sampleCount: number
}

View File

@@ -0,0 +1,102 @@
import { TimerInterface } from '@standardnotes/time'
import { MetricsStoreInterface } from '../../../Metrics/MetricsStoreInterface'
import { CheckForTrafficAbuse } from './CheckForTrafficAbuse'
import { MetricsSummary } from '../../../Metrics/MetricsSummary'
import { Metric } from '../../../Metrics/Metric'
import { Logger } from 'winston'
describe('CheckForTrafficAbuse', () => {
let metricsStore: MetricsStoreInterface
let timer: TimerInterface
let timeframeLengthInMinutes: number
let threshold: number
let logger: Logger
const createUseCase = () => new CheckForTrafficAbuse(metricsStore, timer, logger)
beforeEach(() => {
logger = {} as jest.Mocked<Logger>
logger.debug = jest.fn()
const metricsSummary: MetricsSummary = {
sum: 101,
max: 0,
min: 0,
sampleCount: 0,
}
metricsStore = {} as jest.Mocked<MetricsStoreInterface>
metricsStore.getUserBasedMetricsSummaryWithinTimeRange = jest.fn().mockReturnValue(metricsSummary)
timer = {} as TimerInterface
timer.getTimestampInMicroseconds = jest.fn().mockReturnValue(0)
timer.getUTCDateNMinutesAgo = jest.fn().mockReturnValue(new Date(0))
timer.getUTCDate = jest.fn().mockReturnValue(new Date(10))
timeframeLengthInMinutes = 5
threshold = 100
})
it('returns a failure result if the user uuid is invalid', async () => {
const result = await createUseCase().execute({
userUuid: 'invalid',
metricToCheck: Metric.NAMES.ItemOperation,
timeframeLengthInMinutes,
threshold,
})
expect(result.isFailed()).toBeTruthy()
})
it('return a failure result if the metric name is invalid', async () => {
const result = await createUseCase().execute({
userUuid: '00000000-0000-0000-0000-000000000000',
metricToCheck: 'invalid',
timeframeLengthInMinutes,
threshold,
})
expect(result.isFailed()).toBeTruthy()
})
it('returns a failure result if the metric summary is above the threshold', async () => {
const metricsSummary: MetricsSummary = {
sum: 101,
max: 0,
min: 0,
sampleCount: 0,
}
metricsStore.getUserBasedMetricsSummaryWithinTimeRange = jest.fn().mockReturnValue(metricsSummary)
const result = await createUseCase().execute({
userUuid: '00000000-0000-0000-0000-000000000000',
metricToCheck: Metric.NAMES.ItemOperation,
timeframeLengthInMinutes,
threshold,
})
expect(result.isFailed()).toBeTruthy()
})
it('returns a success result if the metric summary is below the threshold', async () => {
const metricsSummary: MetricsSummary = {
sum: 99,
max: 0,
min: 0,
sampleCount: 0,
}
metricsStore.getUserBasedMetricsSummaryWithinTimeRange = jest.fn().mockReturnValue(metricsSummary)
const result = await createUseCase().execute({
userUuid: '00000000-0000-0000-0000-000000000000',
metricToCheck: Metric.NAMES.ItemOperation,
timeframeLengthInMinutes,
threshold,
})
expect(result.isFailed()).toBeFalsy()
})
})

View File

@@ -0,0 +1,61 @@
import { Result, UseCaseInterface, Uuid } from '@standardnotes/domain-core'
import { TimerInterface } from '@standardnotes/time'
import { CheckForTrafficAbuseDTO } from './CheckForTrafficAbuseDTO'
import { MetricsStoreInterface } from '../../../Metrics/MetricsStoreInterface'
import { Metric } from '../../../Metrics/Metric'
import { MetricsSummary } from '../../../Metrics/MetricsSummary'
import { Logger } from 'winston'
export class CheckForTrafficAbuse implements UseCaseInterface<MetricsSummary> {
constructor(
private metricsStore: MetricsStoreInterface,
private timer: TimerInterface,
private logger: Logger,
) {}
async execute(dto: CheckForTrafficAbuseDTO): Promise<Result<MetricsSummary>> {
this.logger.debug(`Checking for traffic abuse for metric: ${dto.metricToCheck}.`, {
codeTag: 'CheckForTrafficAbuse',
userId: dto.userUuid,
})
const userUuidOrError = Uuid.create(dto.userUuid)
if (userUuidOrError.isFailed()) {
return Result.fail(userUuidOrError.getError())
}
const userUuid = userUuidOrError.getValue()
const metricToCheckOrError = Metric.create({
name: dto.metricToCheck,
timestamp: this.timer.getTimestampInMicroseconds(),
})
if (metricToCheckOrError.isFailed()) {
return Result.fail(metricToCheckOrError.getError())
}
const metricToCheck = metricToCheckOrError.getValue()
const metricsSummary = await this.metricsStore.getUserBasedMetricsSummaryWithinTimeRange({
metricName: metricToCheck.props.name,
userUuid,
from: this.timer.getUTCDateNMinutesAgo(dto.timeframeLengthInMinutes),
to: this.timer.getUTCDate(),
})
this.logger.debug(
`Current traffic abuse metric ${dto.metricToCheck} value in timeframe of ${dto.timeframeLengthInMinutes} minutes is ${metricsSummary.sum}. The threshold is ${dto.threshold}`,
{
codeTag: 'CheckForTrafficAbuse',
userId: dto.userUuid,
},
)
if (metricsSummary.sum > dto.threshold) {
return Result.fail(
`Traffic abuse detected for metric: ${metricToCheck.props.name}. Usage ${metricsSummary.sum} is greater than threshold ${dto.threshold}`,
)
}
return Result.ok(metricsSummary)
}
}

View File

@@ -0,0 +1,6 @@
export interface CheckForTrafficAbuseDTO {
userUuid: string
metricToCheck: string
timeframeLengthInMinutes: number
threshold: number
}

View File

@@ -63,6 +63,7 @@ export class GetItems implements UseCaseInterface<GetItemsResult> {
const { uuids, transferLimitBreachedBeforeEndOfItems } = await this.itemTransferCalculator.computeItemUuidsToFetch(
itemContentSizeDescriptors,
this.contentSizeTransferLimit,
userUuid,
)
let items: Array<Item> = []
if (uuids.length > 0) {

View File

@@ -27,6 +27,7 @@ describe('SaveNewItem', () => {
metricsStore = {} as jest.Mocked<MetricsStoreInterface>
metricsStore.storeMetric = jest.fn()
metricsStore.storeUserBasedMetric = jest.fn()
item1 = Item.create(
{

View File

@@ -110,6 +110,7 @@ export class SaveNewItem implements UseCaseInterface<Item> {
return Result.fail(itemOrError.getError())
}
const newItem = itemOrError.getValue()
newItem.props.contentSize = Buffer.byteLength(JSON.stringify(newItem))
if (dto.itemHash.representsASharedVaultItem()) {
const sharedVaultAssociationOrError = SharedVaultAssociation.create({
@@ -138,6 +139,15 @@ export class SaveNewItem implements UseCaseInterface<Item> {
await this.itemRepository.insert(newItem)
await this.metricsStore.storeUserBasedMetric(
Metric.create({
name: Metric.NAMES.ContentSizeUtilized,
timestamp: this.timer.getTimestampInMicroseconds(),
}).getValue(),
newItem.props.contentSize,
userUuid,
)
await this.metricsStore.storeMetric(
Metric.create({ name: Metric.NAMES.ItemCreated, timestamp: this.timer.getTimestampInMicroseconds() }).getValue(),
)

View File

@@ -53,6 +53,7 @@ describe('UpdateExistingItem', () => {
metricsStore = {} as jest.Mocked<MetricsStoreInterface>
metricsStore.storeMetric = jest.fn()
metricsStore.storeUserBasedMetric = jest.fn()
item1 = Item.create(
{

View File

@@ -180,6 +180,15 @@ export class UpdateExistingItem implements UseCaseInterface<Item> {
Metric.create({ name: Metric.NAMES.ItemUpdated, timestamp: this.timer.getTimestampInMicroseconds() }).getValue(),
)
await this.metricsStore.storeUserBasedMetric(
Metric.create({
name: Metric.NAMES.ContentSizeUtilized,
timestamp: this.timer.getTimestampInMicroseconds(),
}).getValue(),
dto.existingItem.props.contentSize,
userUuid,
)
/* istanbul ignore next */
const revisionsFrequency = dto.isFreeUser ? this.freeRevisionFrequency : this.premiumRevisionFrequency

View File

@@ -1,16 +1,42 @@
import { Uuid } from '@standardnotes/domain-core'
import { MetricsStoreInterface } from '../../Domain/Metrics/MetricsStoreInterface'
import { Metric } from '../../Domain/Metrics/Metric'
import { MetricsSummary } from '../../Domain/Metrics/MetricsSummary'
export class DummyMetricStore implements MetricsStoreInterface {
async getUserBasedMetricsSummaryWithinTimeRange(_dto: {
metricName: string
userUuid: Uuid
from: Date
to: Date
}): Promise<MetricsSummary> {
return {
sum: 0,
max: 0,
min: 0,
sampleCount: 0,
}
}
async getUserBasedMetricsSummary(_name: string, _timestamp: number): Promise<MetricsSummary> {
return {
sum: 0,
max: 0,
min: 0,
sampleCount: 0,
}
}
async storeUserBasedMetric(_metric: Metric, _value: number, _userUuid: Uuid): Promise<void> {
// do nothing
}
async storeMetric(_metric: Metric): Promise<void> {
// do nothing
}
async getStatistics(
_name: string,
_from: number,
_to: number,
): Promise<{ sum: number; max: number; min: number; sampleCount: number }> {
async getMetricsSummary(_name: string, _from: number, _to: number): Promise<MetricsSummary> {
return {
sum: 0,
max: 0,

View File

@@ -11,18 +11,42 @@ import { SyncItems } from '../../Domain/UseCase/Syncing/SyncItems/SyncItems'
import { BaseItemsController } from './Base/BaseItemsController'
import { MapperInterface } from '@standardnotes/domain-core'
import { ItemHttpRepresentation } from '../../Mapping/Http/ItemHttpRepresentation'
import { CheckForTrafficAbuse } from '../../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse'
import { Logger } from 'winston'
@controller('/items', TYPES.Sync_AuthMiddleware)
export class AnnotatedItemsController extends BaseItemsController {
constructor(
@inject(TYPES.Sync_CheckForTrafficAbuse) override checkForTrafficAbuse: CheckForTrafficAbuse,
@inject(TYPES.Sync_SyncItems) override syncItems: SyncItems,
@inject(TYPES.Sync_CheckIntegrity) override checkIntegrity: CheckIntegrity,
@inject(TYPES.Sync_GetItem) override getItem: GetItem,
@inject(TYPES.Sync_ItemHttpMapper) override itemHttpMapper: MapperInterface<Item, ItemHttpRepresentation>,
@inject(TYPES.Sync_SyncResponseFactoryResolver)
override syncResponseFactoryResolver: SyncResponseFactoryResolverInterface,
@inject(TYPES.Sync_Logger) override logger: Logger,
@inject(TYPES.Sync_STRICT_ABUSE_PROTECTION) override strictAbuseProtection: boolean,
@inject(TYPES.Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES)
override itemOperationsAbuseTimeframeLengthInMinutes: number,
@inject(TYPES.Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD) override itemOperationsAbuseThreshold: number,
@inject(TYPES.Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD) override payloadSizeAbuseThreshold: number,
@inject(TYPES.Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES)
override payloadSizeAbuseTimeframeLengthInMinutes: number,
) {
super(syncItems, checkIntegrity, getItem, itemHttpMapper, syncResponseFactoryResolver)
super(
checkForTrafficAbuse,
syncItems,
checkIntegrity,
getItem,
itemHttpMapper,
syncResponseFactoryResolver,
logger,
strictAbuseProtection,
itemOperationsAbuseTimeframeLengthInMinutes,
itemOperationsAbuseThreshold,
payloadSizeAbuseThreshold,
payloadSizeAbuseTimeframeLengthInMinutes,
)
}
@httpPost('/sync')

View File

@@ -11,14 +11,24 @@ import { ApiVersion } from '../../../Domain/Api/ApiVersion'
import { SyncItems } from '../../../Domain/UseCase/Syncing/SyncItems/SyncItems'
import { ItemHttpRepresentation } from '../../../Mapping/Http/ItemHttpRepresentation'
import { ItemHash } from '../../../Domain/Item/ItemHash'
import { CheckForTrafficAbuse } from '../../../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse'
import { Metric } from '../../../Domain/Metrics/Metric'
import { Logger } from 'winston'
export class BaseItemsController extends BaseHttpController {
constructor(
protected checkForTrafficAbuse: CheckForTrafficAbuse,
protected syncItems: SyncItems,
protected checkIntegrity: CheckIntegrity,
protected getItem: GetItem,
protected itemHttpMapper: MapperInterface<Item, ItemHttpRepresentation>,
protected syncResponseFactoryResolver: SyncResponseFactoryResolverInterface,
protected logger: Logger,
protected strictAbuseProtection: boolean,
protected itemOperationsAbuseTimeframeLengthInMinutes: number,
protected itemOperationsAbuseThreshold: number,
protected payloadSizeAbuseThreshold: number,
protected payloadSizeAbuseTimeframeLengthInMinutes: number,
private controllerContainer?: ControllerContainerInterface,
) {
super()
@@ -31,6 +41,37 @@ export class BaseItemsController extends BaseHttpController {
}
async sync(request: Request, response: Response): Promise<results.JsonResult> {
const checkForItemOperationsAbuseResult = await this.checkForTrafficAbuse.execute({
metricToCheck: Metric.NAMES.ItemOperation,
userUuid: response.locals.user.uuid,
threshold: this.itemOperationsAbuseThreshold,
timeframeLengthInMinutes: this.itemOperationsAbuseTimeframeLengthInMinutes,
})
if (checkForItemOperationsAbuseResult.isFailed()) {
this.logger.warn(checkForItemOperationsAbuseResult.getError(), {
userId: response.locals.user.uuid,
})
if (this.strictAbuseProtection) {
return this.json({ error: { message: checkForItemOperationsAbuseResult.getError() } }, 429)
}
}
const checkForPayloadSizeAbuseResult = await this.checkForTrafficAbuse.execute({
metricToCheck: Metric.NAMES.ContentSizeUtilized,
userUuid: response.locals.user.uuid,
threshold: this.payloadSizeAbuseThreshold,
timeframeLengthInMinutes: this.payloadSizeAbuseTimeframeLengthInMinutes,
})
if (checkForPayloadSizeAbuseResult.isFailed()) {
this.logger.warn(checkForPayloadSizeAbuseResult.getError(), {
userId: response.locals.user.uuid,
})
if (this.strictAbuseProtection) {
return this.json({ error: { message: checkForPayloadSizeAbuseResult.getError() } }, 429)
}
}
const itemHashes: ItemHash[] = []
if ('items' in request.body) {
for (const itemHashInput of request.body.items) {

View File

@@ -3,20 +3,142 @@ import { TimerInterface } from '@standardnotes/time'
import { MetricsStoreInterface } from '../../Domain/Metrics/MetricsStoreInterface'
import { Metric } from '../../Domain/Metrics/Metric'
import { Uuid } from '@standardnotes/domain-core'
import { MetricsSummary } from '../../Domain/Metrics/MetricsSummary'
import { Logger } from 'winston'
export class RedisMetricStore implements MetricsStoreInterface {
private readonly METRIC_PREFIX = 'metric'
private readonly METRIC_PER_USER_PREFIX = 'metric-user'
constructor(
private redisClient: IORedis.Redis,
private timer: TimerInterface,
private logger: Logger,
) {}
async getStatistics(
name: string,
from: number,
to: number,
): Promise<{ sum: number; max: number; min: number; sampleCount: number }> {
async getUserBasedMetricsSummaryWithinTimeRange(dto: {
metricName: string
userUuid: Uuid
from: Date
to: Date
}): Promise<MetricsSummary> {
this.logger.debug(`Fetching user based metrics summary for ${dto.metricName}.`, {
codeTag: 'RedisMetricStore',
userId: dto.userUuid.value,
from: dto.from.toISOString(),
to: dto.to.toISOString(),
})
const keys = this.getKeysRepresentingMinutesBetweenFromAndTo(dto.from, dto.to)
this.logger.debug(`Fetching user based metrics summary for ${dto.metricName} - keys: ${keys.join(', ')}.`, {
codeTag: 'RedisMetricStore',
userId: dto.userUuid.value,
})
let sum = 0
let max = 0
let min = 0
let sampleCount = 0
const values = await this.redisClient.mget(
keys.map((key) => `${this.METRIC_PER_USER_PREFIX}:${dto.userUuid.value}:${dto.metricName}:${key}`),
)
this.logger.debug(`Fetching user based metrics summary for ${dto.metricName} - values: ${values.join(', ')}.`, {
codeTag: 'RedisMetricStore',
userId: dto.userUuid.value,
})
for (const value of values) {
if (!value) {
continue
}
const valueAsNumber = Number(value)
sum += valueAsNumber
sampleCount++
if (valueAsNumber > max) {
max = valueAsNumber
}
if (valueAsNumber < min) {
min = valueAsNumber
}
}
return {
sum,
max,
min,
sampleCount,
}
}
async storeUserBasedMetric(metric: Metric, value: number, userUuid: Uuid): Promise<void> {
const date = this.timer.convertMicrosecondsToDate(metric.props.timestamp)
const dateToTheMinuteString = this.timer.convertDateToFormattedString(date, 'YYYY-MM-DD HH:mm')
const key = `${this.METRIC_PER_USER_PREFIX}:${userUuid.value}:${metric.props.name}:${dateToTheMinuteString}`
const pipeline = this.redisClient.pipeline()
pipeline.incrbyfloat(key, value)
pipeline.incr(
`${this.METRIC_PER_USER_PREFIX}:${userUuid.value}:${Metric.NAMES.ItemOperation}:${dateToTheMinuteString}`,
)
const expirationTime = 60 * 60 * 24
pipeline.expire(key, expirationTime)
await pipeline.exec()
}
async getUserBasedMetricsSummary(name: string, timestamp: number): Promise<MetricsSummary> {
const date = this.timer.convertMicrosecondsToDate(timestamp)
const dateToTheMinuteString = this.timer.convertDateToFormattedString(date, 'YYYY-MM-DD HH:mm')
const userMetricsKeys = await this.redisClient.keys(
`${this.METRIC_PER_USER_PREFIX}:*:${name}:${dateToTheMinuteString}`,
)
let sum = 0
let max = 0
let min = 0
let sampleCount = 0
const values = await this.redisClient.mget(userMetricsKeys)
for (const value of values) {
if (!value) {
continue
}
const valueAsNumber = Number(value)
sum += valueAsNumber
sampleCount++
if (valueAsNumber > max) {
max = valueAsNumber
}
if (valueAsNumber < min) {
min = valueAsNumber
}
}
return {
sum,
max,
min,
sampleCount,
}
}
async getMetricsSummary(name: string, from: number, to: number): Promise<MetricsSummary> {
const keysRepresentingSecondsBetweenFromAndTo = this.getKeysRepresentingSecondsBetweenFromAndTo(from, to)
let sum = 0
@@ -64,12 +186,28 @@ export class RedisMetricStore implements MetricsStoreInterface {
pipeline.incr(key)
const expirationTimeIn24Hours = 60 * 60 * 24
pipeline.expire(key, expirationTimeIn24Hours)
const expirationTime = 60 * 60 * 6
pipeline.expire(key, expirationTime)
await pipeline.exec()
}
private getKeysRepresentingMinutesBetweenFromAndTo(from: Date, to: Date): string[] {
const keys: string[] = []
let currentMinute = from
while (currentMinute <= to) {
const dateToTheMinuteString = this.timer.convertDateToFormattedString(currentMinute, 'YYYY-MM-DD HH:mm')
keys.push(dateToTheMinuteString)
currentMinute = new Date(currentMinute.getTime() + 60 * 1000)
}
return keys
}
private getKeysRepresentingSecondsBetweenFromAndTo(from: number, to: number): string[] {
const keys: string[] = []

View File

@@ -9,12 +9,20 @@ import { SyncItems } from '../../Domain/UseCase/Syncing/SyncItems/SyncItems'
import { ApiVersion } from '../../Domain/Api/ApiVersion'
import { SyncResponseFactoryResolverInterface } from '../../Domain/Item/SyncResponse/SyncResponseFactoryResolverInterface'
import { SyncResponse20200115 } from '../../Domain/Item/SyncResponse/SyncResponse20200115'
import { CheckForTrafficAbuse } from '../../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse'
import { Metric } from '../../Domain/Metrics/Metric'
export class SyncingServer implements ISyncingServer {
constructor(
private syncItemsUseCase: SyncItems,
private syncResponseFactoryResolver: SyncResponseFactoryResolverInterface,
private mapper: MapperInterface<SyncResponse20200115, SyncResponse>,
protected checkForTrafficAbuse: CheckForTrafficAbuse,
private strictAbuseProtection: boolean,
private itemOperationsAbuseTimeframeLengthInMinutes: number,
private itemOperationsAbuseThreshold: number,
private payloadSizeAbuseThreshold: number,
private payloadSizeAbuseTimeframeLengthInMinutes: number,
private logger: Logger,
) {}
@@ -23,7 +31,62 @@ export class SyncingServer implements ISyncingServer {
callback: grpc.sendUnaryData<SyncResponse>,
): Promise<void> {
try {
this.logger.debug('[SyncingServer] Syncing items via gRPC')
const userUuid = call.metadata.get('x-user-uuid').pop() as string
const checkForItemOperationsAbuseResult = await this.checkForTrafficAbuse.execute({
metricToCheck: Metric.NAMES.ItemOperation,
userUuid,
threshold: this.itemOperationsAbuseThreshold,
timeframeLengthInMinutes: this.itemOperationsAbuseTimeframeLengthInMinutes,
})
if (checkForItemOperationsAbuseResult.isFailed()) {
this.logger.warn(checkForItemOperationsAbuseResult.getError(), {
userId: userUuid,
})
if (this.strictAbuseProtection) {
const metadata = new grpc.Metadata()
metadata.set('x-sync-error-message', checkForItemOperationsAbuseResult.getError())
metadata.set('x-sync-error-response-code', '429')
return callback(
{
code: Status.INVALID_ARGUMENT,
message: checkForItemOperationsAbuseResult.getError(),
name: 'INVALID_ARGUMENT',
metadata,
},
null,
)
}
}
const checkForPayloadSizeAbuseResult = await this.checkForTrafficAbuse.execute({
metricToCheck: Metric.NAMES.ContentSizeUtilized,
userUuid,
threshold: this.payloadSizeAbuseThreshold,
timeframeLengthInMinutes: this.payloadSizeAbuseTimeframeLengthInMinutes,
})
if (checkForPayloadSizeAbuseResult.isFailed()) {
this.logger.warn(checkForPayloadSizeAbuseResult.getError(), {
userId: userUuid,
})
if (this.strictAbuseProtection) {
const metadata = new grpc.Metadata()
metadata.set('x-sync-error-message', checkForPayloadSizeAbuseResult.getError())
metadata.set('x-sync-error-response-code', '429')
return callback(
{
code: Status.INVALID_ARGUMENT,
message: checkForPayloadSizeAbuseResult.getError(),
name: 'INVALID_ARGUMENT',
metadata,
},
null,
)
}
}
const itemHashesRPC = call.request.getItemsList()
const itemHashes: ItemHash[] = []
@@ -41,7 +104,7 @@ export class SyncingServer implements ISyncingServer {
created_at_timestamp: itemHash.hasCreatedAtTimestamp() ? itemHash.getCreatedAtTimestamp() : undefined,
updated_at: itemHash.hasUpdatedAt() ? itemHash.getUpdatedAt() : undefined,
updated_at_timestamp: itemHash.hasUpdatedAtTimestamp() ? itemHash.getUpdatedAtTimestamp() : undefined,
user_uuid: call.metadata.get('userUuid').pop() as string,
user_uuid: userUuid,
key_system_identifier: itemHash.hasKeySystemIdentifier()
? (itemHash.getKeySystemIdentifier() as string)
: null,
@@ -74,7 +137,6 @@ export class SyncingServer implements ISyncingServer {
}
const apiVersion = call.request.hasApiVersion() ? (call.request.getApiVersion() as string) : ApiVersion.v20161215
const userUuid = call.metadata.get('x-user-uuid').pop() as string
const readOnlyAccess = call.metadata.get('x-read-only-access').pop() === 'true'
if (readOnlyAccess) {
this.logger.debug('Syncing with read-only access', {