Compare commits

...

20 Commits

Author SHA1 Message Date
Tom Moor 5c938b0d1a Use retention period presets instead of arbitrary values
Replace free-form retention days with a fixed set of presets
(0, 7, 14, 30, 90, 180, 365) where 0 means infinite/never delete.
This bounds the number of cron sub-tasks and simplifies the system
by removing findUniquePreferenceValues in favor of iterating over
known presets directly.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-29 17:30:46 -04:00
Tom Moor e7220c47f7 Merge main 2026-03-29 16:58:19 -04:00
Tom Moor 0a0e3cd677 lint 2025-12-20 11:57:09 -05:00
Tom Moor 11a30943b2 willPermanentlyDeleteAt 2025-12-20 11:45:50 -05:00
Tom Moor dfd28c5787 Update app/components/DocumentListItem.tsx
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-12-20 11:45:22 -05:00
Tom Moor e63d876174 Use updateWithCtx 2025-12-20 11:29:15 -05:00
Tom Moor 217ac4eb43 tsc 2025-12-20 11:20:29 -05:00
Tom Moor 96cff654f6 Remove literal usage 2025-12-20 11:15:19 -05:00
Tom Moor b9c1181bd3 findUniquePreferenceValues method 2025-12-20 10:56:09 -05:00
Tom Moor 0b9df2658b stash 2025-12-20 10:42:51 -05:00
Tom Moor f937de00ff stash 2025-12-20 10:30:24 -05:00
Tom Moor 4f4ef31e19 refactor 2025-12-20 10:10:52 -05:00
Tom Moor 944f162f91 Merge public/main 2025-12-20 08:41:25 -05:00
Tom Moor dc4c2785b2 tsc 2025-12-18 22:24:35 -05:00
Tom Moor e26c337ed3 Account for custom retention in UI 2025-12-18 22:19:00 -05:00
Tom Moor 6a96ce4f8e test 2025-12-18 20:57:37 -05:00
Tom Moor b281ab503e interpolation 2025-12-18 20:50:05 -05:00
Tom Moor bc93c6d059 test 2025-12-18 20:42:28 -05:00
Tom Moor d4d0144564 split cron 2025-12-18 20:32:54 -05:00
Tom Moor 9988ae0143 wip 2025-12-18 19:36:28 -05:00
30 changed files with 971 additions and 260 deletions
+21 -1
View File
@@ -40,6 +40,8 @@ type Props = {
showCollection?: boolean;
showPublished?: boolean;
showDraft?: boolean;
showLastViewed?: boolean;
showTemplate?: boolean;
};
const SEARCH_RESULT_REGEX = /<b\b[^>]*>(.*?)<\/b>/gi;
@@ -76,6 +78,8 @@ function DocumentListItem(
showCollection,
showPublished,
showDraft = true,
showLastViewed = true,
showTemplate,
highlight,
context,
...rest
@@ -176,7 +180,7 @@ function DocumentListItem(
showCollection={showCollection}
showPublished={showPublished}
showParentDocuments={showParentDocuments}
showLastViewed
showLastViewed={showLastViewed}
/>
</Content>
</Flex>
@@ -187,6 +191,22 @@ function DocumentListItem(
onClose={handleMenuClose}
/>
</Actions>
{document.deletedAt &&
document.permanentlyDeletesInDays &&
document.permanentlyDeletesInDays >= 0 && (
<Tooltip
content={t("Permanently deletes in {{ days }} days", {
days: document.permanentlyDeletesInDays,
})}
placement="bottom"
>
<Badge>
{t("{{ days }} days", {
days: document.permanentlyDeletesInDays,
})}
</Badge>
</Tooltip>
)}
</DocumentLink>
</ContextMenu>
</ActionContextProvider>
+3
View File
@@ -15,6 +15,7 @@ type Props = {
showCollection?: boolean;
showPublished?: boolean;
showDraft?: boolean;
showLastViewed?: boolean;
showTemplate?: boolean;
};
@@ -29,6 +30,7 @@ const PaginatedDocumentList = React.memo<Props>(function PaginatedDocumentList({
showPublished,
showTemplate,
showDraft,
showLastViewed,
...rest
}: Props) {
const { t } = useTranslation();
@@ -50,6 +52,7 @@ const PaginatedDocumentList = React.memo<Props>(function PaginatedDocumentList({
showCollection={showCollection}
showPublished={showPublished}
showDraft={showDraft}
showLastViewed={showLastViewed}
/>
)}
{...rest}
+30 -2
View File
@@ -13,6 +13,7 @@ import {
FileOperationFormat,
NavigationNodeType,
NotificationEventType,
TeamPreference,
} from "@shared/types";
import Storage from "@shared/utils/Storage";
import { isRTL } from "@shared/utils/rtl";
@@ -195,6 +196,9 @@ export default class Document extends ArchivableModel implements Searchable {
@observable
publishedAt: string | undefined;
@observable
permanentlyDeletedAt: string | undefined;
@observable
popularityScore: number;
@@ -395,12 +399,36 @@ export default class Document extends ArchivableModel implements Searchable {
}
@computed
get permanentlyDeletedAt(): string | undefined {
get willPermanentlyDeleteAt(): string | undefined {
if (!this.deletedAt) {
return undefined;
}
return addDays(new Date(this.deletedAt), 30).toString();
const team = this.store.rootStore.auth.team;
if (!team) {
return undefined;
}
const retentionDays = team.getPreference(
TeamPreference.TrashRetentionDays,
30
);
if (!retentionDays) {
return undefined;
}
return addDays(new Date(this.deletedAt), retentionDays).toString();
}
@computed
get permanentlyDeletesInDays(): number | undefined {
if (!this.deletedAt) {
return undefined;
}
return differenceInDays(
new Date(this.willPermanentlyDeleteAt!),
new Date()
);
}
@computed
+3 -3
View File
@@ -28,7 +28,7 @@ export default function Notices({ document }: Props) {
const { t } = useTranslation();
function permanentlyDeletedDescription() {
if (!document.permanentlyDeletedAt) {
if (!document.willPermanentlyDeleteAt) {
return;
}
@@ -36,9 +36,9 @@ export default function Notices({ document }: Props) {
// to avoid showing a negative number of days. The cleanup task will
// permanently delete the document at the next run.
const permanentlyDeletedAt =
new Date(document.permanentlyDeletedAt) < new Date()
new Date(document.willPermanentlyDeleteAt) < new Date()
? new Date().toISOString()
: document.permanentlyDeletedAt;
: document.willPermanentlyDeleteAt;
return (
<Trans>
+5
View File
@@ -34,8 +34,13 @@ function Trash() {
<PaginatedDocumentList
documents={documents.deleted}
fetch={documents.fetchDeleted}
options={{
direction: "asc",
sort: "deletedAt",
}}
heading={<Subheading sticky>{t("Recently deleted")}</Subheading>}
empty={<Empty>{t("Trash is empty at the moment.")}</Empty>}
showLastViewed={false}
showCollection
showTemplate
/>
+2 -2
View File
@@ -210,8 +210,8 @@ export default class DocumentsStore extends Store<Document> {
@computed
get deleted(): Document[] {
return orderBy(this.orderedData, "deletedAt", "desc").filter(
(d) => d.deletedAt
return orderBy(this.orderedData, "deletedAt", "asc").filter(
(d) => d.deletedAt && !d.permanentlyDeletedAt
);
}
@@ -0,0 +1,22 @@
"use strict";
module.exports = {
async up(queryInterface, Sequelize) {
await queryInterface.addColumn("documents", "permanentlyDeletedAt", {
type: Sequelize.DATE,
allowNull: true,
});
await queryInterface.addIndex("documents", ["permanentlyDeletedAt"], {
name: "documents_permanently_deleted_at",
concurrently: true,
});
},
async down(queryInterface) {
await queryInterface.removeIndex(
"documents",
"documents_permanently_deleted_at"
);
await queryInterface.removeColumn("documents", "permanentlyDeletedAt");
},
};
+12
View File
@@ -22,6 +22,7 @@ import {
BeforeValidate,
BeforeCreate,
BeforeUpdate,
BeforeRestore,
HasMany,
BeforeSave,
DefaultScope,
@@ -390,6 +391,17 @@ class Document extends ArchivableModel<
@Column
publishedAt: Date | null;
/** Whether the document has been permanently deleted (softly), and if so when. */
@AllowNull
@IsDate
@Column
permanentlyDeletedAt: Date | null;
@BeforeRestore
static clearPermanentlyDeletedAt(model: Document) {
model.permanentlyDeletedAt = null;
}
/** An array of user IDs that have edited this document. */
@Column(DataType.ARRAY(DataType.UUID))
collaboratorIds: string[] = [];
+17 -11
View File
@@ -1,5 +1,9 @@
import { randomUUID } from "node:crypto";
import { buildTeam, buildCollection, buildAttachment } from "@server/test/factories";
import {
buildTeam,
buildCollection,
buildAttachment,
} from "@server/test/factories";
describe("Team", () => {
describe("collectionIds", () => {
@@ -23,22 +27,24 @@ describe("Team", () => {
describe("previousSubdomains", () => {
it("should list the previous subdomains", async () => {
const s1 = `example-${Math.random().toString(36).substring(7)}`;
const s2 = `updated-${Math.random().toString(36).substring(7)}`;
const s3 = `another-${Math.random().toString(36).substring(7)}`;
const team = await buildTeam({
subdomain: "example",
subdomain: s1,
});
const subdomain = "updated";
await team.update({ subdomain });
expect(team.subdomain).toEqual(subdomain);
await team.update({ subdomain: s2 });
expect(team.subdomain).toEqual(s2);
expect(team.previousSubdomains?.length).toEqual(1);
expect(team.previousSubdomains?.[0]).toEqual("example");
expect(team.previousSubdomains?.[0]).toEqual(s1);
const subdomain2 = "another";
await team.update({ subdomain: subdomain2 });
expect(team.subdomain).toEqual(subdomain2);
await team.update({ subdomain: s3 });
expect(team.subdomain).toEqual(s3);
expect(team.previousSubdomains?.length).toEqual(2);
expect(team.previousSubdomains?.[0]).toEqual("example");
expect(team.previousSubdomains?.[1]).toEqual(subdomain);
expect(team.previousSubdomains?.[0]).toEqual(s1);
expect(team.previousSubdomains?.[1]).toEqual(s2);
});
});
+1
View File
@@ -77,6 +77,7 @@ async function presentDocument(
publishedAt: document.publishedAt,
archivedAt: document.archivedAt,
deletedAt: document.deletedAt,
permanentlyDeletedAt: document.permanentlyDeletedAt,
collaboratorIds: [],
revision: document.revisionCount,
fullWidth: document.fullWidth,
@@ -1,97 +0,0 @@
import { subDays } from "date-fns";
import { Document } from "@server/models";
import { buildDocument, buildTeam } from "@server/test/factories";
import CleanupDeletedDocumentsTask from "./CleanupDeletedDocumentsTask";
const props = {
limit: 100,
partition: {
partitionIndex: 0,
partitionCount: 1,
},
};
describe("CleanupDeletedDocumentsTask", () => {
it("should not destroy documents not deleted", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
});
const task = new CleanupDeletedDocumentsTask();
await task.perform(props);
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(1);
});
it("should not destroy documents deleted less than 30 days ago", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 25),
});
const task = new CleanupDeletedDocumentsTask();
await task.perform(props);
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(1);
});
it("should destroy documents deleted more than 30 days ago", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 60),
});
const task = new CleanupDeletedDocumentsTask();
await task.perform(props);
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(0);
});
it("should destroy draft documents deleted more than 30 days ago", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: undefined,
deletedAt: subDays(new Date(), 60),
});
const task = new CleanupDeletedDocumentsTask();
await task.perform(props);
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(0);
});
});
@@ -0,0 +1,138 @@
import { subDays } from "date-fns";
import { Document } from "@server/models";
import { buildDocument, buildTeam } from "@server/test/factories";
import { TeamPreferenceDefaults } from "@shared/constants";
import { TeamPreference } from "@shared/types";
import CleanupPermanentlyDeletedDocumentsByRetentionTask from "./CleanupPermanentlyDeletedDocumentsByRetentionTask";
const props = {
limit: 100,
partition: {
partitionIndex: 0,
partitionCount: 1,
},
};
const defaultRetentionDays = TeamPreferenceDefaults[
TeamPreference.DataRetentionDays
] as number;
describe("CleanupPermanentlyDeletedDocumentsByRetentionTask", () => {
it("should not destroy documents not marked for permanent deletion", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 60),
permanentlyDeletedAt: null,
});
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
await task.perform({ ...props, retentionDays: defaultRetentionDays });
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(1);
});
it("should not destroy documents marked for permanent deletion less than default retention days ago", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 60),
permanentlyDeletedAt: subDays(new Date(), defaultRetentionDays - 5),
});
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
await task.perform({ ...props, retentionDays: defaultRetentionDays });
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(1);
});
it("should destroy documents marked for permanent deletion more than default retention days ago", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 60),
permanentlyDeletedAt: subDays(new Date(), defaultRetentionDays + 1),
});
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
await task.perform({ ...props, retentionDays: defaultRetentionDays });
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(0);
});
it("should respect custom documentRetentionDays", async () => {
const retentionDays = 7;
const team = await buildTeam();
team.setPreference(TeamPreference.DataRetentionDays, retentionDays);
await team.save();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 60),
permanentlyDeletedAt: subDays(new Date(), 10),
});
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
await task.perform({ ...props, retentionDays });
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(0);
});
it("should not destroy documents if within custom documentRetentionDays", async () => {
const retentionDays = 90;
const team = await buildTeam();
team.setPreference(TeamPreference.DataRetentionDays, retentionDays);
await team.save();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 60),
permanentlyDeletedAt: subDays(new Date(), 45),
});
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
await task.perform({ ...props, retentionDays });
expect(
await Document.unscoped().count({
where: {
teamId: team.id,
},
paranoid: false,
})
).toEqual(1);
});
});
@@ -0,0 +1,88 @@
import { Op, Sequelize } from "sequelize";
import { subDays } from "date-fns";
import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
import Logger from "@server/logging/Logger";
import { Document } from "@server/models";
import { TeamPreferenceDefaults } from "@shared/constants";
import { TeamPreference } from "@shared/types";
import { BaseTask, TaskPriority } from "./base/BaseTask";
import type { PartitionInfo } from "./base/BaseTask";
export type Props = {
/** The retention period in days to process in this tranche. */
retentionDays: number;
/** The maximum number of documents to destroy in this task. */
limit: number;
/** Partition information for distributing work. */
partition?: PartitionInfo;
};
/**
* A task that handles the permanent destruction of documents past their retention period.
*/
export default class CleanupPermanentlyDeletedDocumentsByRetentionTask extends BaseTask<Props> {
public async perform({ limit, partition, retentionDays }: Props) {
// Infinite retention means documents are never permanently deleted.
if (retentionDays === 0) {
return;
}
const defaultRetentionDays = TeamPreferenceDefaults[
TeamPreference.DataRetentionDays
] as number;
const isDefault = retentionDays === defaultRetentionDays;
Logger.debug(
"task",
`Permanently destroying upto ${limit} documents past ${retentionDays} day retention timeout…`
);
const documents = await Document.scope([
"withDrafts",
"withoutState",
]).findAll({
where: {
permanentlyDeletedAt: {
[Op.lt]: subDays(new Date(), retentionDays),
},
[Op.and]: [
Sequelize.literal(
isDefault
? `EXISTS (
SELECT 1 FROM teams
WHERE teams.id = "document"."teamId"
AND (
preferences->> :preference IS NULL
OR (preferences->> :preference)::int = :retentionDays
)
)`
: `EXISTS (
SELECT 1 FROM teams
WHERE teams.id = "document"."teamId"
AND (preferences->> :preference)::int = :retentionDays
)`
),
],
...this.getPartitionWhereClause("id", partition),
},
replacements: {
preference: TeamPreference.DataRetentionDays,
retentionDays,
},
paranoid: false,
limit,
});
if (documents.length > 0) {
const countDeletedDocument = await documentPermanentDeleter(documents);
Logger.info("task", `Destroyed ${countDeletedDocument} documents`);
}
}
public get options() {
return {
attempts: 1,
priority: TaskPriority.Background,
};
}
}
@@ -0,0 +1,72 @@
import { buildTeam } from "@server/test/factories";
import { TeamPreferenceDefaults } from "@shared/constants";
import { TeamPreference } from "@shared/types";
import CleanupPermanentlyDeletedDocumentsTask from "./CleanupPermanentlyDeletedDocumentsTask";
import CleanupPermanentlyDeletedDocumentsByRetentionTask from "./CleanupPermanentlyDeletedDocumentsByRetentionTask";
const props = {
limit: 10000,
partition: {
partitionIndex: 0,
partitionCount: 1,
},
};
const defaultRetentionDays = TeamPreferenceDefaults[
TeamPreference.DataRetentionDays
] as number;
describe("CleanupPermanentlyDeletedDocumentsTask", () => {
it("should schedule worker tasks for default and custom retention periods", async () => {
const scheduleSpy = jest.spyOn(
CleanupPermanentlyDeletedDocumentsByRetentionTask.prototype,
"schedule"
);
// Team with custom retention
const teamCustom = await buildTeam();
const customDays = 7;
teamCustom.setPreference(TeamPreference.DataRetentionDays, customDays);
await teamCustom.save();
const task = new CleanupPermanentlyDeletedDocumentsTask();
await task.perform(props);
// Verify that the default retention task was scheduled
expect(scheduleSpy).toHaveBeenCalledWith(
expect.objectContaining({
retentionDays: defaultRetentionDays,
partition: props.partition,
})
);
// Verify that the custom retention task was scheduled
expect(scheduleSpy).toHaveBeenCalledWith(
expect.objectContaining({
retentionDays: customDays,
partition: props.partition,
})
);
scheduleSpy.mockRestore();
});
it("should always schedule a worker for the default retention period", async () => {
const scheduleSpy = jest.spyOn(
CleanupPermanentlyDeletedDocumentsByRetentionTask.prototype,
"schedule"
);
const task = new CleanupPermanentlyDeletedDocumentsTask();
await task.perform(props);
expect(scheduleSpy).toHaveBeenCalledWith(
expect.objectContaining({
retentionDays: defaultRetentionDays,
partition: props.partition,
})
);
scheduleSpy.mockRestore();
});
});
@@ -0,0 +1,48 @@
import Logger from "@server/logging/Logger";
import { RetentionPeriodPresets } from "@shared/constants";
import { Minute } from "@shared/utils/time";
import { TaskPriority } from "./base/BaseTask";
import { CronTask, TaskInterval } from "./base/CronTask";
import type { Props } from "./base/CronTask";
import CleanupPermanentlyDeletedDocumentsByRetentionTask from "./CleanupPermanentlyDeletedDocumentsByRetentionTask";
export default class CleanupPermanentlyDeletedDocumentsTask extends CronTask {
/**
* Schedules a worker task for each retention period preset.
*
* @param props Properties to be used by the task.
*/
public async perform(props: Props) {
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
for (const days of RetentionPeriodPresets) {
if (days === 0) {
continue;
}
await task.schedule({
limit: props.limit,
retentionDays: days,
partition: props.partition,
});
}
Logger.debug(
"task",
`Scheduled ${RetentionPeriodPresets.length - 1} tranches for document cleanup`
);
}
public get options() {
return {
attempts: 1,
priority: TaskPriority.Background,
};
}
public get cron() {
return {
interval: TaskInterval.Hour,
partitionWindow: 15 * Minute.ms,
};
}
}
+16 -13
View File
@@ -1,5 +1,4 @@
import { Op } from "sequelize";
import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
import { Document } from "@server/models";
import { BaseTask } from "./base/BaseTask";
@@ -12,18 +11,22 @@ export default class EmptyTrashTask extends BaseTask<Props> {
if (!documentIds.length) {
return;
}
const documents = await Document.unscoped().findAll({
where: {
id: {
[Op.in]: documentIds,
},
// for safety, ensure the documents are in soft-delete state.
deletedAt: {
[Op.ne]: null,
},
await Document.unscoped().update(
{
permanentlyDeletedAt: new Date(),
},
paranoid: false,
});
await documentPermanentDeleter(documents);
{
where: {
id: {
[Op.in]: documentIds,
},
// for safety, ensure the documents are in soft-delete state.
deletedAt: {
[Op.ne]: null,
},
},
paranoid: false,
}
);
}
}
@@ -0,0 +1,116 @@
import { subDays } from "date-fns";
import { Document } from "@server/models";
import { buildDocument, buildTeam } from "@server/test/factories";
import { TeamPreferenceDefaults } from "@shared/constants";
import { TeamPreference } from "@shared/types";
import ExpireDocumentsInTrashByRetentionTask from "./ExpireDocumentsInTrashByRetentionTask";
const props = {
partition: {
partitionIndex: 0,
partitionCount: 1,
},
};
const defaultRetentionDays = TeamPreferenceDefaults[
TeamPreference.TrashRetentionDays
] as number;
describe("ExpireDocumentsInTrashByRetentionTask", () => {
it("should not mark active documents", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
});
const task = new ExpireDocumentsInTrashByRetentionTask();
await task.perform({ ...props, retentionDays: defaultRetentionDays });
const doc = await Document.unscoped().findOne({
where: { teamId: team.id },
paranoid: false,
});
expect(doc?.permanentlyDeletedAt).toBeNull();
});
it("should not mark documents deleted less than 30 days ago (default)", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), defaultRetentionDays - 5),
});
const task = new ExpireDocumentsInTrashByRetentionTask();
await task.perform({ ...props, retentionDays: defaultRetentionDays });
const doc = await Document.unscoped().findOne({
where: { teamId: team.id },
paranoid: false,
});
expect(doc?.permanentlyDeletedAt).toBeNull();
});
it("should mark documents deleted more than 30 days ago (default)", async () => {
const team = await buildTeam();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), defaultRetentionDays + 1),
});
const task = new ExpireDocumentsInTrashByRetentionTask();
await task.perform({ ...props, retentionDays: defaultRetentionDays });
const doc = await Document.unscoped().findOne({
where: { teamId: team.id },
paranoid: false,
});
expect(doc?.permanentlyDeletedAt).not.toBeNull();
});
it("should respect custom trashRetentionDays", async () => {
const retentionDays = 7;
const team = await buildTeam();
team.setPreference(TeamPreference.TrashRetentionDays, retentionDays);
await team.save();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 10),
});
const task = new ExpireDocumentsInTrashByRetentionTask();
await task.perform({ ...props, retentionDays });
const doc = await Document.unscoped().findOne({
where: { teamId: team.id },
paranoid: false,
});
expect(doc?.permanentlyDeletedAt).not.toBeNull();
});
it("should not mark documents if within custom trashRetentionDays", async () => {
const retentionDays = 90;
const team = await buildTeam();
team.setPreference(TeamPreference.TrashRetentionDays, retentionDays);
await team.save();
await buildDocument({
teamId: team.id,
publishedAt: new Date(),
deletedAt: subDays(new Date(), 60),
});
const task = new ExpireDocumentsInTrashByRetentionTask();
await task.perform({ ...props, retentionDays });
const doc = await Document.unscoped().findOne({
where: { teamId: team.id },
paranoid: false,
});
expect(doc?.permanentlyDeletedAt).toBeNull();
});
});
@@ -0,0 +1,88 @@
import { Op, Sequelize } from "sequelize";
import { subDays } from "date-fns";
import Logger from "@server/logging/Logger";
import { Document } from "@server/models";
import { TeamPreferenceDefaults } from "@shared/constants";
import { TeamPreference } from "@shared/types";
import type { PartitionInfo } from "./base/BaseTask";
import { BaseTask, TaskPriority } from "./base/BaseTask";
export type Props = {
/** The trash retention period in days to process in this tranche. */
retentionDays: number;
/** Partition information for distributing work. */
partition?: PartitionInfo;
};
/**
* A task that marks documents in the trash for permanent deletion based on a retention period.
*/
export default class ExpireDocumentsInTrashByRetentionTask extends BaseTask<Props> {
public async perform(props: Props) {
const { partition, retentionDays } = props;
// Infinite retention means documents are never expired from trash.
if (retentionDays === 0) {
return;
}
const defaultTrashRetentionDays = TeamPreferenceDefaults[
TeamPreference.TrashRetentionDays
] as number;
const isDefault = retentionDays === defaultTrashRetentionDays;
Logger.debug(
"task",
`Marking documents past ${retentionDays} day trash timeout as pending permanent deletion…`
);
// Mark documents that have been in the trash for longer than the retention period.
// This moves them from "Trash" to "Pending Permanent Deletion" (Retention phase).
const [count] = await Document.unscoped().update(
{
permanentlyDeletedAt: new Date(),
},
{
where: {
deletedAt: {
[Op.lt]: subDays(new Date(), retentionDays),
},
permanentlyDeletedAt: {
[Op.is]: null,
},
[Op.and]: [
Sequelize.literal(
isDefault
? `EXISTS (
SELECT 1 FROM teams
WHERE teams.id = "documents"."teamId"
AND (
preferences->>'${TeamPreference.TrashRetentionDays}' IS NULL
OR (preferences->>'${TeamPreference.TrashRetentionDays}')::int = ${defaultTrashRetentionDays}
)
)`
: `EXISTS (
SELECT 1 FROM teams
WHERE teams.id = "documents"."teamId"
AND (preferences->>'${TeamPreference.TrashRetentionDays}')::int = ${retentionDays}
)`
),
],
...this.getPartitionWhereClause("id", partition),
},
paranoid: false,
}
);
if (count > 0) {
Logger.info("task", `Marked ${count} documents for permanent deletion`);
}
}
public get options() {
return {
attempts: 1,
priority: TaskPriority.Background,
};
}
}
@@ -0,0 +1,72 @@
import { buildTeam } from "@server/test/factories";
import { TeamPreferenceDefaults } from "@shared/constants";
import { TeamPreference } from "@shared/types";
import ExpireDocumentsInTrashTask from "./ExpireDocumentsInTrashTask";
import ExpireDocumentsInTrashByRetentionTask from "./ExpireDocumentsInTrashByRetentionTask";
const props = {
limit: 100,
partition: {
partitionIndex: 0,
partitionCount: 1,
},
};
const defaultRetentionDays = TeamPreferenceDefaults[
TeamPreference.TrashRetentionDays
] as number;
describe("ExpireDocumentsInTrashTask", () => {
it("should schedule worker tasks for default and custom retention periods", async () => {
const scheduleSpy = jest.spyOn(
ExpireDocumentsInTrashByRetentionTask.prototype,
"schedule"
);
// Team with custom retention
const teamCustom = await buildTeam();
const customDays = 7;
teamCustom.setPreference(TeamPreference.TrashRetentionDays, customDays);
await teamCustom.save();
const task = new ExpireDocumentsInTrashTask();
await task.perform(props);
// Verify that the default retention task was scheduled
expect(scheduleSpy).toHaveBeenCalledWith(
expect.objectContaining({
retentionDays: defaultRetentionDays,
partition: props.partition,
})
);
// Verify that the custom retention task was scheduled.
expect(scheduleSpy).toHaveBeenCalledWith(
expect.objectContaining({
retentionDays: customDays,
partition: props.partition,
})
);
scheduleSpy.mockRestore();
});
it("should always schedule a worker for the default retention period", async () => {
const scheduleSpy = jest.spyOn(
ExpireDocumentsInTrashByRetentionTask.prototype,
"schedule"
);
const task = new ExpireDocumentsInTrashTask();
await task.perform(props);
expect(scheduleSpy).toHaveBeenCalledWith(
expect.objectContaining({
retentionDays: defaultRetentionDays,
partition: props.partition,
})
);
scheduleSpy.mockRestore();
});
});
@@ -0,0 +1,47 @@
import Logger from "@server/logging/Logger";
import { RetentionPeriodPresets } from "@shared/constants";
import { Minute } from "@shared/utils/time";
import { TaskPriority } from "./base/BaseTask";
import { CronTask, TaskInterval } from "./base/CronTask";
import type { Props } from "./base/CronTask";
import ExpireDocumentsInTrashByRetentionTask from "./ExpireDocumentsInTrashByRetentionTask";
export default class ExpireDocumentsInTrashTask extends CronTask {
/**
* Schedules a worker task for each retention period preset.
*
* @param props Properties to be used by the task.
*/
public async perform(props: Props) {
const task = new ExpireDocumentsInTrashByRetentionTask();
for (const days of RetentionPeriodPresets) {
if (days === 0) {
continue;
}
await task.schedule({
retentionDays: days,
partition: props.partition,
});
}
Logger.debug(
"task",
`Scheduled ${RetentionPeriodPresets.length - 1} tranches for marking documents for permanent deletion`
);
}
public get options() {
return {
attempts: 1,
priority: TaskPriority.Background,
};
}
public get cron() {
return {
interval: TaskInterval.Hour,
partitionWindow: 15 * Minute.ms,
};
}
}
@@ -5,8 +5,8 @@ import { QueryTypes } from "sequelize";
import { Minute } from "@shared/utils/time";
import env from "@server/env";
import Logger from "@server/logging/Logger";
import { TaskPriority } from "./base/BaseTask";
import type { PartitionInfo, Props } from "./base/CronTask";
import { type PartitionInfo, TaskPriority } from "./base/BaseTask";
import type { Props } from "./base/CronTask";
import { CronTask, TaskInterval } from "./base/CronTask";
import { sequelize, sequelizeReadOnly } from "@server/storage/database";
+117
View File
@@ -1,5 +1,7 @@
import type { Job, JobOptions } from "bull";
import { taskQueue } from "../../";
import type { WhereOptions } from "sequelize";
import { Op } from "sequelize";
export enum TaskPriority {
Background = 40,
@@ -8,6 +10,20 @@ export enum TaskPriority {
High = 10,
}
/**
* Partition information for distributing work across multiple worker instances.
*/
export type PartitionInfo = {
/**
* The partition number for this task instance (0-based).
*/
partitionIndex: number;
/**
* The total number of partitions.
*/
partitionCount: number;
};
export abstract class BaseTask<T extends Record<string, any>> {
/**
* Schedule this task type to be processed asynchronously by a worker.
@@ -58,4 +74,105 @@ export abstract class BaseTask<T extends Record<string, any>> {
},
};
}
/**
* Optimized partitioning method for UUID primary keys using range-based distribution.
* Divides the UUID space into N equal ranges and assigns each partition a range.
*
* The UUID space (0x00000000-... to 0xffffffff-...) is divided into N equal ranges.
* For example, with 3 partitions:
* - Partition 0: '00000000-0000-4000-8000-000000000000' to '55555554-ffff-4fff-bfff-ffffffffffff'
* - Partition 1: '55555555-0000-4000-8000-000000000000' to 'aaaaaaaa-ffff-4fff-bfff-ffffffffffff'
* - Partition 2: 'aaaaaaab-0000-4000-8000-000000000000' to 'ffffffff-ffff-4fff-bfff-ffffffffffff'
*
* @param partitionInfo The partition information
* @returns The start and end UUID bounds for the partition
*/
protected getPartitionBounds(
partitionInfo: PartitionInfo | undefined
): [string, string] {
if (!partitionInfo) {
return [
"00000000-0000-4000-8000-000000000000",
"ffffffff-ffff-4fff-bfff-ffffffffffff",
];
}
const { partitionIndex, partitionCount } = partitionInfo;
if (
partitionCount <= 0 ||
partitionIndex < 0 ||
partitionIndex >= partitionCount
) {
throw new Error(
`Invalid partition info: index ${partitionIndex}, count ${partitionCount}`
);
}
// 2^32 total possible values for the first 32 bits (4.3 billion)
const TOTAL_VALUES = 0x100000000;
// The maximum possible integer value (0xFFFFFFFF)
const MAX_VALUE = TOTAL_VALUES - 1;
// Ensure even distribution of values by calculating exact range size
const rangeSize = Math.floor(TOTAL_VALUES / partitionCount);
const rangeStart = partitionIndex * rangeSize;
let rangeEnd: number;
if (partitionIndex === partitionCount - 1) {
// The last partition takes any remainder and goes up to the max value
rangeEnd = MAX_VALUE;
} else {
// The end is the start of the *next* partition minus 1
rangeEnd = (partitionIndex + 1) * rangeSize - 1;
}
// Use Number.prototype.toString(16) and padStart(8, '0') for the 32-bit hex prefix
const startHex = rangeStart.toString(16).padStart(8, "0");
const endHex = rangeEnd.toString(16).padStart(8, "0");
// Start: First 32 bits (prefix) followed by the lowest possible values for the rest
// Ensures correct UUID v4 version (4xxx) and variant (8|9|a|bxxx) bits
const startUuid = `${startHex}-0000-4000-8000-000000000000`;
// End: First 32 bits (prefix) followed by the highest possible values for the rest
// Ensures correct UUID v4 version (4xxx) and variant (8|9|a|bxxx) bits
const endUuid = `${endHex}-ffff-4fff-bfff-ffffffffffff`;
return [startUuid, endUuid];
}
/**
* Optimized partitioning method for UUID primary keys using range-based distribution.
* Divides the UUID space into N equal ranges and assigns each partition a range.
*
* @param idField The name of the UUID primary key field to partition on
* @param partitionInfo The partition information
* @returns A WHERE clause for partitioned queries
*
* @example
* const where = {
* deletedAt: { [Op.lt]: someDate },
* ...this.getPartitionWhereClause("id", props.partition)
* };
*/
protected getPartitionWhereClause(
idField: string,
partitionInfo: PartitionInfo | undefined
): WhereOptions {
if (!partitionInfo) {
return {};
}
const [startUuid, endUuid] = this.getPartitionBounds(partitionInfo);
return {
[idField]: {
[Op.gte]: startUuid,
[Op.lte]: endUuid,
},
};
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
import { Op } from "sequelize";
import type { PartitionInfo } from "./CronTask";
import type { PartitionInfo } from "./BaseTask";
import { CronTask, TaskInterval } from "./CronTask";
// Create a concrete implementation of CronTask for testing
+2 -118
View File
@@ -1,5 +1,4 @@
import type { WhereOptions } from "sequelize";
import { Op } from "sequelize";
import type { PartitionInfo } from "./BaseTask";
import { BaseTask } from "./BaseTask";
export enum TaskInterval {
@@ -35,20 +34,6 @@ export type TaskSchedule = {
partitionWindow?: number;
};
/**
* Partition information for distributing work across multiple worker instances.
*/
export type PartitionInfo = {
/**
* The partition number for this task instance (0-based).
*/
partitionIndex: number;
/**
* The total number of partitions.
*/
partitionCount: number;
};
/**
* Properties for cron-scheduled tasks.
*/
@@ -57,108 +42,7 @@ export type Props = {
partition: PartitionInfo;
};
export abstract class CronTask extends BaseTask<Props> {
export abstract class CronTask<P extends Props = Props> extends BaseTask<P> {
/** The schedule configuration for this cron task */
public abstract get cron(): TaskSchedule;
/**
* Optimized partitioning method for UUID primary keys using range-based distribution.
* Divides the UUID space into N equal ranges and assigns each partition a range.
*
* The UUID space (0x00000000-... to 0xffffffff-...) is divided into N equal ranges.
* For example, with 3 partitions:
* - Partition 0: '00000000-0000-4000-8000-000000000000' to '55555554-ffff-4fff-bfff-ffffffffffff'
* - Partition 1: '55555555-0000-4000-8000-000000000000' to 'aaaaaaaa-ffff-4fff-bfff-ffffffffffff'
* - Partition 2: 'aaaaaaab-0000-4000-8000-000000000000' to 'ffffffff-ffff-4fff-bfff-ffffffffffff'
*
* @param partitionInfo The partition information
* @returns The start and end UUID bounds for the partition
*/
protected getPartitionBounds(
partitionInfo: PartitionInfo | undefined
): [string, string] {
if (!partitionInfo) {
return [
"00000000-0000-4000-8000-000000000000",
"ffffffff-ffff-4fff-bfff-ffffffffffff",
];
}
const { partitionIndex, partitionCount } = partitionInfo;
if (
partitionCount <= 0 ||
partitionIndex < 0 ||
partitionIndex >= partitionCount
) {
throw new Error(
`Invalid partition info: index ${partitionIndex}, count ${partitionCount}`
);
}
// 2^32 total possible values for the first 32 bits (4.3 billion)
const TOTAL_VALUES = 0x100000000;
// The maximum possible integer value (0xFFFFFFFF)
const MAX_VALUE = TOTAL_VALUES - 1;
// Ensure even distribution of values by calculating exact range size
const rangeSize = Math.floor(TOTAL_VALUES / partitionCount);
const rangeStart = partitionIndex * rangeSize;
let rangeEnd: number;
if (partitionIndex === partitionCount - 1) {
// The last partition takes any remainder and goes up to the max value
rangeEnd = MAX_VALUE;
} else {
// The end is the start of the *next* partition minus 1
rangeEnd = (partitionIndex + 1) * rangeSize - 1;
}
// Use Number.prototype.toString(16) and padStart(8, '0') for the 32-bit hex prefix
const startHex = rangeStart.toString(16).padStart(8, "0");
const endHex = rangeEnd.toString(16).padStart(8, "0");
// Start: First 32 bits (prefix) followed by the lowest possible values for the rest
// Ensures correct UUID v4 version (4xxx) and variant (8|9|a|bxxx) bits
const startUuid = `${startHex}-0000-4000-8000-000000000000`;
// End: First 32 bits (prefix) followed by the highest possible values for the rest
// Ensures correct UUID v4 version (4xxx) and variant (8|9|a|bxxx) bits
const endUuid = `${endHex}-ffff-4fff-bfff-ffffffffffff`;
return [startUuid, endUuid];
}
/**
* Optimized partitioning method for UUID primary keys using range-based distribution.
* Divides the UUID space into N equal ranges and assigns each partition a range.
*
* @param idField The name of the UUID primary key field to partition on
* @param partitionInfo The partition information
* @returns A WHERE clause for partitioned queries
*
* @example
* const where = {
* deletedAt: { [Op.lt]: someDate },
* ...this.getPartitionWhereClause("id", props.partition)
* };
*/
protected getPartitionWhereClause(
idField: string,
partitionInfo: PartitionInfo | undefined
): WhereOptions {
if (!partitionInfo) {
return {};
}
const [startUuid, endUuid] = this.getPartitionBounds(partitionInfo);
return {
[idField]: {
[Op.gte]: startUuid,
[Op.lte]: endUuid,
},
};
}
}
+8 -9
View File
@@ -28,7 +28,6 @@ import documentCreator from "@server/commands/documentCreator";
import documentDuplicator from "@server/commands/documentDuplicator";
import documentLoader from "@server/commands/documentLoader";
import documentMover from "@server/commands/documentMover";
import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
import documentUpdater from "@server/commands/documentUpdater";
import env from "@server/env";
import {
@@ -405,6 +404,9 @@ router.post(
deletedAt: {
[Op.ne]: null,
},
permanentlyDeletedAt: {
[Op.is]: null,
},
[Op.or]: [
{
collectionId: {
@@ -1477,14 +1479,8 @@ router.post(
});
authorize(user, "permanentDelete", document);
await documentPermanentDeleter([document]);
await Event.createFromContext(ctx, {
name: "documents.permanent_delete",
documentId: document.id,
collectionId: document.collectionId,
data: {
title: document.title,
},
await document.updateWithCtx(ctx, {
permanentlyDeletedAt: new Date(),
});
} else {
const document = await Document.findByPk(id, {
@@ -2097,6 +2093,9 @@ router.post(
deletedAt: {
[Op.ne]: null,
},
permanentlyDeletedAt: {
[Op.is]: null,
},
[Op.or]: [
{
collectionId: {
+1
View File
@@ -20,6 +20,7 @@ const DocumentsSortParamsSchema = z.object({
[
"createdAt",
"updatedAt",
"deletedAt",
"publishedAt",
"index",
"title",
+16
View File
@@ -3,6 +3,18 @@ import { EmailDisplay, TOCPosition, UserRole } from "@shared/types";
import { TeamValidation } from "@shared/validations";
import { BaseSchema } from "@server/routes/api/schema";
const retentionDaysSchema = z
.union([
z.literal(0),
z.literal(7),
z.literal(14),
z.literal(30),
z.literal(90),
z.literal(180),
z.literal(365),
])
.optional();
export const TeamsUpdateSchema = BaseSchema.extend({
body: z.object({
/** Team name */
@@ -66,6 +78,10 @@ export const TeamsUpdateSchema = BaseSchema.extend({
emailDisplay: z.enum(EmailDisplay).optional(),
/** Whether to prevent shared documents from being embedded in iframes on external websites. */
preventDocumentEmbedding: z.boolean().optional(),
/** Days to keep documents in trash before retention phase. 0 = infinite. */
trashRetentionDays: retentionDaysSchema,
/** Days to keep documents in retention phase before permanent deletion. 0 = infinite. */
dataRetentionDays: retentionDaysSchema,
/** Whether external MCP clients can connect to the workspace. */
mcp: z.boolean().optional(),
/** List of disabled embed provider titles. */
+12 -1
View File
@@ -1,4 +1,8 @@
import type { TeamPreferences, UserPreferences } from "./types";
import type {
RetentionPeriodPreset,
TeamPreferences,
UserPreferences,
} from "./types";
import {
TOCPosition,
TeamPreference,
@@ -7,6 +11,11 @@ import {
NotificationBadgeType,
} from "./types";
/** Allowed retention period values in days. 0 means infinite (never delete). */
export const RetentionPeriodPresets: readonly RetentionPeriodPreset[] = [
0, 7, 14, 30, 90, 180, 365,
];
export const MAX_AVATAR_DISPLAY = 6;
export const Pagination = {
@@ -34,6 +43,8 @@ export const TeamPreferenceDefaults: TeamPreferences = {
[TeamPreference.CustomTheme]: undefined,
[TeamPreference.TocPosition]: TOCPosition.Left,
[TeamPreference.PreventDocumentEmbedding]: false,
[TeamPreference.TrashRetentionDays]: 30,
[TeamPreference.DataRetentionDays]: 30,
[TeamPreference.EmailDisplay]: EmailDisplay.Members,
[TeamPreference.MCP]: true,
[TeamPreference.DisabledEmbeds]: [],
@@ -272,6 +272,8 @@
"New": "New",
"Only visible to you": "Only visible to you",
"Draft": "Draft",
"Permanently deletes in {{ days }} days": "Permanently deletes in {{ days }} days",
"{{ days }} days": "{{ days }} days",
"You updated": "You updated",
"{{ userName }} updated": "{{ userName }} updated",
"You deleted": "You deleted",
+9
View File
@@ -1,3 +1,6 @@
/** Allowed retention period in days. 0 means infinite (never delete). */
export type RetentionPeriodPreset = 0 | 7 | 14 | 30 | 90 | 180 | 365;
/** Available user roles. */
export enum UserRole {
Admin = "admin",
@@ -406,6 +409,10 @@ export enum TeamPreference {
TocPosition = "tocPosition",
/** Whether to prevent shared documents from being embedded in iframes on external websites. */
PreventDocumentEmbedding = "preventDocumentEmbedding",
/** The number of days to keep documents in the trash before moving to the retention phase. */
TrashRetentionDays = "trashRetentionDays",
/** The number of days to keep documents in the retention phase before permanent deletion. */
DataRetentionDays = "dataRetentionDays",
/** Who can see user email addresses. */
EmailDisplay = "emailDisplay",
/** Whether external MCP clients can connect to the workspace. */
@@ -426,6 +433,8 @@ export type TeamPreferences = {
[TeamPreference.CustomTheme]?: Partial<CustomTheme>;
[TeamPreference.TocPosition]?: TOCPosition;
[TeamPreference.PreventDocumentEmbedding]?: boolean;
[TeamPreference.TrashRetentionDays]?: RetentionPeriodPreset;
[TeamPreference.DataRetentionDays]?: RetentionPeriodPreset;
[TeamPreference.EmailDisplay]?: EmailDisplay;
[TeamPreference.MCP]?: boolean;
[TeamPreference.DisabledEmbeds]?: string[];