mylesmarkevich/standardnotes-server
1
0
Fork 0
mirror of https://github.com/standardnotes/server synced 2026-10-16 18:13:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(syncing-server): revisions processing limit

Browse Source
This commit is contained in:
Karol Sójko
2022-12-15 10:38:25 +01:00
parent 1f20395ff3
commit f10fa839fb
1 changed files with 30 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/syncing-server/bin/revisions.ts
+30 -8
View File
@@ -15,6 +15,7 @@ import { ContentType } from '@standardnotes/common'
const fixRevisionsOwnership = async (
year: number,
month: number,
revisionsProcessingLimit: number,
itemRepository: ItemRepositoryInterface,
domainEventFactory: DomainEventFactoryInterface,
domainEventPublisher: DomainEventPublisherInterface,
@@ -23,7 +24,7 @@ const fixRevisionsOwnership = async (
const createdAfter = new Date(`${year}-${month}-1`)
const createdBefore = new Date(`${month !== 12 ? year : year + 1}-${month !== 12 ? month + 1 : 1}-1`)
logger.info(`Processing items between ${createdAfter.toISOString()} and ${createdBefore.toISOString()}`)
logger.info(`[${createdAfter.toISOString()} - ${createdBefore.toISOString()}] Processing items`)
const itemsCount = await itemRepository.countAll({
createdBetween: [createdAfter, createdBefore],
@@ -33,17 +34,21 @@ const fixRevisionsOwnership = async (
sortBy: 'uuid',
})
logger.info(`There are ${itemsCount} items between ${createdAfter.toISOString()} and ${createdBefore.toISOString()}`)
logger.info(
`[${createdAfter.toISOString()} - ${createdBefore.toISOString()}] There are ${itemsCount} items to process.`,
)
const limit = 500
const amountOfPages = Math.ceil(itemsCount / limit)
const amountOfPages = Math.ceil(itemsCount / revisionsProcessingLimit)
const tenPercentOfPages = Math.ceil(amountOfPages / 10)
let itemsProcessedCounter = 0
let itemsSkippedCounter = 0
for (let page = 1; page <= amountOfPages; page++) {
if (page % tenPercentOfPages === 0) {
logger.info(
`Processing page ${page} of ${amountOfPages} items between ${createdAfter.toISOString()} and ${createdBefore.toISOString()}. Processed successfully ${itemsProcessedCounter} items. Skipped ${itemsSkippedCounter} items.`,
`[${createdAfter.toISOString()} - ${createdBefore.toISOString()}] Processing page ${page}/${amountOfPages} of items.`,
)
logger.info(
`[${createdAfter.toISOString()} - ${createdBefore.toISOString()}] Processed successfully/skipped items: ${itemsProcessedCounter}/${itemsSkippedCounter}.`,
)
}
@@ -51,12 +56,20 @@ const fixRevisionsOwnership = async (
createdBetween: [createdAfter, createdBefore],
selectFields: ['uuid', 'user_uuid'],
contentType: [ContentType.Note, ContentType.File],
offset: (page - 1) * limit,
limit,
offset: (page - 1) * revisionsProcessingLimit,
limit: revisionsProcessingLimit,
sortOrder: 'ASC',
sortBy: 'uuid',
})
if (items.length === 0) {
logger.warn(
`[${createdAfter.toISOString()} - ${createdBefore.toISOString()}] No items fetched for offset ${
(page - 1) * revisionsProcessingLimit
} and limit ${revisionsProcessingLimit}.`,
)
}
for (const item of items) {
if (!item.userUuid || !item.uuid) {
itemsSkippedCounter++
@@ -90,12 +103,21 @@ void container.load().then((container) => {
const years = env.get('REVISION_YEARS').split(',')
const months = env.get('REVISION_MONTHS').split(',')
const revisionsProcessingLimit = env.get('REVISIONS_PROCESSING_LIMIT')
const promises = []
for (const year of years) {
for (const month of months) {
promises.push(
fixRevisionsOwnership(+year, +month, itemRepository, domainEventFactory, domainEventPublisher, logger),
fixRevisionsOwnership(
+year,
+month,
+revisionsProcessingLimit,
itemRepository,
domainEventFactory,
domainEventPublisher,
logger,
),
)
}
}