mirror of
https://github.com/outline/outline.git
synced 2026-06-13 11:25:03 +03:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5c938b0d1a | |||
| e7220c47f7 | |||
| 0a0e3cd677 | |||
| 11a30943b2 | |||
| dfd28c5787 | |||
| e63d876174 | |||
| 217ac4eb43 | |||
| 96cff654f6 | |||
| b9c1181bd3 | |||
| 0b9df2658b | |||
| f937de00ff | |||
| 4f4ef31e19 | |||
| 944f162f91 | |||
| dc4c2785b2 | |||
| e26c337ed3 | |||
| 6a96ce4f8e | |||
| b281ab503e | |||
| bc93c6d059 | |||
| d4d0144564 | |||
| 9988ae0143 |
@@ -40,6 +40,8 @@ type Props = {
|
||||
showCollection?: boolean;
|
||||
showPublished?: boolean;
|
||||
showDraft?: boolean;
|
||||
showLastViewed?: boolean;
|
||||
showTemplate?: boolean;
|
||||
};
|
||||
|
||||
const SEARCH_RESULT_REGEX = /<b\b[^>]*>(.*?)<\/b>/gi;
|
||||
@@ -76,6 +78,8 @@ function DocumentListItem(
|
||||
showCollection,
|
||||
showPublished,
|
||||
showDraft = true,
|
||||
showLastViewed = true,
|
||||
showTemplate,
|
||||
highlight,
|
||||
context,
|
||||
...rest
|
||||
@@ -176,7 +180,7 @@ function DocumentListItem(
|
||||
showCollection={showCollection}
|
||||
showPublished={showPublished}
|
||||
showParentDocuments={showParentDocuments}
|
||||
showLastViewed
|
||||
showLastViewed={showLastViewed}
|
||||
/>
|
||||
</Content>
|
||||
</Flex>
|
||||
@@ -187,6 +191,22 @@ function DocumentListItem(
|
||||
onClose={handleMenuClose}
|
||||
/>
|
||||
</Actions>
|
||||
{document.deletedAt &&
|
||||
document.permanentlyDeletesInDays &&
|
||||
document.permanentlyDeletesInDays >= 0 && (
|
||||
<Tooltip
|
||||
content={t("Permanently deletes in {{ days }} days", {
|
||||
days: document.permanentlyDeletesInDays,
|
||||
})}
|
||||
placement="bottom"
|
||||
>
|
||||
<Badge>
|
||||
{t("{{ days }} days", {
|
||||
days: document.permanentlyDeletesInDays,
|
||||
})}
|
||||
</Badge>
|
||||
</Tooltip>
|
||||
)}
|
||||
</DocumentLink>
|
||||
</ContextMenu>
|
||||
</ActionContextProvider>
|
||||
|
||||
@@ -15,6 +15,7 @@ type Props = {
|
||||
showCollection?: boolean;
|
||||
showPublished?: boolean;
|
||||
showDraft?: boolean;
|
||||
showLastViewed?: boolean;
|
||||
showTemplate?: boolean;
|
||||
};
|
||||
|
||||
@@ -29,6 +30,7 @@ const PaginatedDocumentList = React.memo<Props>(function PaginatedDocumentList({
|
||||
showPublished,
|
||||
showTemplate,
|
||||
showDraft,
|
||||
showLastViewed,
|
||||
...rest
|
||||
}: Props) {
|
||||
const { t } = useTranslation();
|
||||
@@ -50,6 +52,7 @@ const PaginatedDocumentList = React.memo<Props>(function PaginatedDocumentList({
|
||||
showCollection={showCollection}
|
||||
showPublished={showPublished}
|
||||
showDraft={showDraft}
|
||||
showLastViewed={showLastViewed}
|
||||
/>
|
||||
)}
|
||||
{...rest}
|
||||
|
||||
+30
-2
@@ -13,6 +13,7 @@ import {
|
||||
FileOperationFormat,
|
||||
NavigationNodeType,
|
||||
NotificationEventType,
|
||||
TeamPreference,
|
||||
} from "@shared/types";
|
||||
import Storage from "@shared/utils/Storage";
|
||||
import { isRTL } from "@shared/utils/rtl";
|
||||
@@ -195,6 +196,9 @@ export default class Document extends ArchivableModel implements Searchable {
|
||||
@observable
|
||||
publishedAt: string | undefined;
|
||||
|
||||
@observable
|
||||
permanentlyDeletedAt: string | undefined;
|
||||
|
||||
@observable
|
||||
popularityScore: number;
|
||||
|
||||
@@ -395,12 +399,36 @@ export default class Document extends ArchivableModel implements Searchable {
|
||||
}
|
||||
|
||||
@computed
|
||||
get permanentlyDeletedAt(): string | undefined {
|
||||
get willPermanentlyDeleteAt(): string | undefined {
|
||||
if (!this.deletedAt) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return addDays(new Date(this.deletedAt), 30).toString();
|
||||
const team = this.store.rootStore.auth.team;
|
||||
if (!team) {
|
||||
return undefined;
|
||||
}
|
||||
const retentionDays = team.getPreference(
|
||||
TeamPreference.TrashRetentionDays,
|
||||
30
|
||||
);
|
||||
if (!retentionDays) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return addDays(new Date(this.deletedAt), retentionDays).toString();
|
||||
}
|
||||
|
||||
@computed
|
||||
get permanentlyDeletesInDays(): number | undefined {
|
||||
if (!this.deletedAt) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return differenceInDays(
|
||||
new Date(this.willPermanentlyDeleteAt!),
|
||||
new Date()
|
||||
);
|
||||
}
|
||||
|
||||
@computed
|
||||
|
||||
@@ -28,7 +28,7 @@ export default function Notices({ document }: Props) {
|
||||
const { t } = useTranslation();
|
||||
|
||||
function permanentlyDeletedDescription() {
|
||||
if (!document.permanentlyDeletedAt) {
|
||||
if (!document.willPermanentlyDeleteAt) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -36,9 +36,9 @@ export default function Notices({ document }: Props) {
|
||||
// to avoid showing a negative number of days. The cleanup task will
|
||||
// permanently delete the document at the next run.
|
||||
const permanentlyDeletedAt =
|
||||
new Date(document.permanentlyDeletedAt) < new Date()
|
||||
new Date(document.willPermanentlyDeleteAt) < new Date()
|
||||
? new Date().toISOString()
|
||||
: document.permanentlyDeletedAt;
|
||||
: document.willPermanentlyDeleteAt;
|
||||
|
||||
return (
|
||||
<Trans>
|
||||
|
||||
@@ -34,8 +34,13 @@ function Trash() {
|
||||
<PaginatedDocumentList
|
||||
documents={documents.deleted}
|
||||
fetch={documents.fetchDeleted}
|
||||
options={{
|
||||
direction: "asc",
|
||||
sort: "deletedAt",
|
||||
}}
|
||||
heading={<Subheading sticky>{t("Recently deleted")}</Subheading>}
|
||||
empty={<Empty>{t("Trash is empty at the moment.")}</Empty>}
|
||||
showLastViewed={false}
|
||||
showCollection
|
||||
showTemplate
|
||||
/>
|
||||
|
||||
@@ -210,8 +210,8 @@ export default class DocumentsStore extends Store<Document> {
|
||||
|
||||
@computed
|
||||
get deleted(): Document[] {
|
||||
return orderBy(this.orderedData, "deletedAt", "desc").filter(
|
||||
(d) => d.deletedAt
|
||||
return orderBy(this.orderedData, "deletedAt", "asc").filter(
|
||||
(d) => d.deletedAt && !d.permanentlyDeletedAt
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"use strict";
|
||||
|
||||
module.exports = {
|
||||
async up(queryInterface, Sequelize) {
|
||||
await queryInterface.addColumn("documents", "permanentlyDeletedAt", {
|
||||
type: Sequelize.DATE,
|
||||
allowNull: true,
|
||||
});
|
||||
await queryInterface.addIndex("documents", ["permanentlyDeletedAt"], {
|
||||
name: "documents_permanently_deleted_at",
|
||||
concurrently: true,
|
||||
});
|
||||
},
|
||||
|
||||
async down(queryInterface) {
|
||||
await queryInterface.removeIndex(
|
||||
"documents",
|
||||
"documents_permanently_deleted_at"
|
||||
);
|
||||
await queryInterface.removeColumn("documents", "permanentlyDeletedAt");
|
||||
},
|
||||
};
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
BeforeValidate,
|
||||
BeforeCreate,
|
||||
BeforeUpdate,
|
||||
BeforeRestore,
|
||||
HasMany,
|
||||
BeforeSave,
|
||||
DefaultScope,
|
||||
@@ -390,6 +391,17 @@ class Document extends ArchivableModel<
|
||||
@Column
|
||||
publishedAt: Date | null;
|
||||
|
||||
/** Whether the document has been permanently deleted (softly), and if so when. */
|
||||
@AllowNull
|
||||
@IsDate
|
||||
@Column
|
||||
permanentlyDeletedAt: Date | null;
|
||||
|
||||
@BeforeRestore
|
||||
static clearPermanentlyDeletedAt(model: Document) {
|
||||
model.permanentlyDeletedAt = null;
|
||||
}
|
||||
|
||||
/** An array of user IDs that have edited this document. */
|
||||
@Column(DataType.ARRAY(DataType.UUID))
|
||||
collaboratorIds: string[] = [];
|
||||
|
||||
+17
-11
@@ -1,5 +1,9 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { buildTeam, buildCollection, buildAttachment } from "@server/test/factories";
|
||||
import {
|
||||
buildTeam,
|
||||
buildCollection,
|
||||
buildAttachment,
|
||||
} from "@server/test/factories";
|
||||
|
||||
describe("Team", () => {
|
||||
describe("collectionIds", () => {
|
||||
@@ -23,22 +27,24 @@ describe("Team", () => {
|
||||
|
||||
describe("previousSubdomains", () => {
|
||||
it("should list the previous subdomains", async () => {
|
||||
const s1 = `example-${Math.random().toString(36).substring(7)}`;
|
||||
const s2 = `updated-${Math.random().toString(36).substring(7)}`;
|
||||
const s3 = `another-${Math.random().toString(36).substring(7)}`;
|
||||
|
||||
const team = await buildTeam({
|
||||
subdomain: "example",
|
||||
subdomain: s1,
|
||||
});
|
||||
const subdomain = "updated";
|
||||
|
||||
await team.update({ subdomain });
|
||||
expect(team.subdomain).toEqual(subdomain);
|
||||
await team.update({ subdomain: s2 });
|
||||
expect(team.subdomain).toEqual(s2);
|
||||
expect(team.previousSubdomains?.length).toEqual(1);
|
||||
expect(team.previousSubdomains?.[0]).toEqual("example");
|
||||
expect(team.previousSubdomains?.[0]).toEqual(s1);
|
||||
|
||||
const subdomain2 = "another";
|
||||
await team.update({ subdomain: subdomain2 });
|
||||
expect(team.subdomain).toEqual(subdomain2);
|
||||
await team.update({ subdomain: s3 });
|
||||
expect(team.subdomain).toEqual(s3);
|
||||
expect(team.previousSubdomains?.length).toEqual(2);
|
||||
expect(team.previousSubdomains?.[0]).toEqual("example");
|
||||
expect(team.previousSubdomains?.[1]).toEqual(subdomain);
|
||||
expect(team.previousSubdomains?.[0]).toEqual(s1);
|
||||
expect(team.previousSubdomains?.[1]).toEqual(s2);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -77,6 +77,7 @@ async function presentDocument(
|
||||
publishedAt: document.publishedAt,
|
||||
archivedAt: document.archivedAt,
|
||||
deletedAt: document.deletedAt,
|
||||
permanentlyDeletedAt: document.permanentlyDeletedAt,
|
||||
collaboratorIds: [],
|
||||
revision: document.revisionCount,
|
||||
fullWidth: document.fullWidth,
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
import { subDays } from "date-fns";
|
||||
import { Document } from "@server/models";
|
||||
import { buildDocument, buildTeam } from "@server/test/factories";
|
||||
import CleanupDeletedDocumentsTask from "./CleanupDeletedDocumentsTask";
|
||||
|
||||
const props = {
|
||||
limit: 100,
|
||||
partition: {
|
||||
partitionIndex: 0,
|
||||
partitionCount: 1,
|
||||
},
|
||||
};
|
||||
|
||||
describe("CleanupDeletedDocumentsTask", () => {
|
||||
it("should not destroy documents not deleted", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
});
|
||||
|
||||
const task = new CleanupDeletedDocumentsTask();
|
||||
await task.perform(props);
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(1);
|
||||
});
|
||||
|
||||
it("should not destroy documents deleted less than 30 days ago", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 25),
|
||||
});
|
||||
|
||||
const task = new CleanupDeletedDocumentsTask();
|
||||
await task.perform(props);
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(1);
|
||||
});
|
||||
|
||||
it("should destroy documents deleted more than 30 days ago", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 60),
|
||||
});
|
||||
|
||||
const task = new CleanupDeletedDocumentsTask();
|
||||
await task.perform(props);
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(0);
|
||||
});
|
||||
|
||||
it("should destroy draft documents deleted more than 30 days ago", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: undefined,
|
||||
deletedAt: subDays(new Date(), 60),
|
||||
});
|
||||
|
||||
const task = new CleanupDeletedDocumentsTask();
|
||||
await task.perform(props);
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,138 @@
|
||||
import { subDays } from "date-fns";
|
||||
import { Document } from "@server/models";
|
||||
import { buildDocument, buildTeam } from "@server/test/factories";
|
||||
import { TeamPreferenceDefaults } from "@shared/constants";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import CleanupPermanentlyDeletedDocumentsByRetentionTask from "./CleanupPermanentlyDeletedDocumentsByRetentionTask";
|
||||
|
||||
const props = {
|
||||
limit: 100,
|
||||
partition: {
|
||||
partitionIndex: 0,
|
||||
partitionCount: 1,
|
||||
},
|
||||
};
|
||||
|
||||
const defaultRetentionDays = TeamPreferenceDefaults[
|
||||
TeamPreference.DataRetentionDays
|
||||
] as number;
|
||||
|
||||
describe("CleanupPermanentlyDeletedDocumentsByRetentionTask", () => {
|
||||
it("should not destroy documents not marked for permanent deletion", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 60),
|
||||
permanentlyDeletedAt: null,
|
||||
});
|
||||
|
||||
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays: defaultRetentionDays });
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(1);
|
||||
});
|
||||
|
||||
it("should not destroy documents marked for permanent deletion less than default retention days ago", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 60),
|
||||
permanentlyDeletedAt: subDays(new Date(), defaultRetentionDays - 5),
|
||||
});
|
||||
|
||||
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays: defaultRetentionDays });
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(1);
|
||||
});
|
||||
|
||||
it("should destroy documents marked for permanent deletion more than default retention days ago", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 60),
|
||||
permanentlyDeletedAt: subDays(new Date(), defaultRetentionDays + 1),
|
||||
});
|
||||
|
||||
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays: defaultRetentionDays });
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(0);
|
||||
});
|
||||
|
||||
it("should respect custom documentRetentionDays", async () => {
|
||||
const retentionDays = 7;
|
||||
const team = await buildTeam();
|
||||
team.setPreference(TeamPreference.DataRetentionDays, retentionDays);
|
||||
await team.save();
|
||||
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 60),
|
||||
permanentlyDeletedAt: subDays(new Date(), 10),
|
||||
});
|
||||
|
||||
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays });
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(0);
|
||||
});
|
||||
|
||||
it("should not destroy documents if within custom documentRetentionDays", async () => {
|
||||
const retentionDays = 90;
|
||||
const team = await buildTeam();
|
||||
team.setPreference(TeamPreference.DataRetentionDays, retentionDays);
|
||||
await team.save();
|
||||
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 60),
|
||||
permanentlyDeletedAt: subDays(new Date(), 45),
|
||||
});
|
||||
|
||||
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays });
|
||||
|
||||
expect(
|
||||
await Document.unscoped().count({
|
||||
where: {
|
||||
teamId: team.id,
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
).toEqual(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
import { Op, Sequelize } from "sequelize";
|
||||
import { subDays } from "date-fns";
|
||||
import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { Document } from "@server/models";
|
||||
import { TeamPreferenceDefaults } from "@shared/constants";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import { BaseTask, TaskPriority } from "./base/BaseTask";
|
||||
import type { PartitionInfo } from "./base/BaseTask";
|
||||
|
||||
export type Props = {
|
||||
/** The retention period in days to process in this tranche. */
|
||||
retentionDays: number;
|
||||
/** The maximum number of documents to destroy in this task. */
|
||||
limit: number;
|
||||
/** Partition information for distributing work. */
|
||||
partition?: PartitionInfo;
|
||||
};
|
||||
|
||||
/**
|
||||
* A task that handles the permanent destruction of documents past their retention period.
|
||||
*/
|
||||
export default class CleanupPermanentlyDeletedDocumentsByRetentionTask extends BaseTask<Props> {
|
||||
public async perform({ limit, partition, retentionDays }: Props) {
|
||||
// Infinite retention means documents are never permanently deleted.
|
||||
if (retentionDays === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const defaultRetentionDays = TeamPreferenceDefaults[
|
||||
TeamPreference.DataRetentionDays
|
||||
] as number;
|
||||
const isDefault = retentionDays === defaultRetentionDays;
|
||||
|
||||
Logger.debug(
|
||||
"task",
|
||||
`Permanently destroying upto ${limit} documents past ${retentionDays} day retention timeout…`
|
||||
);
|
||||
|
||||
const documents = await Document.scope([
|
||||
"withDrafts",
|
||||
"withoutState",
|
||||
]).findAll({
|
||||
where: {
|
||||
permanentlyDeletedAt: {
|
||||
[Op.lt]: subDays(new Date(), retentionDays),
|
||||
},
|
||||
[Op.and]: [
|
||||
Sequelize.literal(
|
||||
isDefault
|
||||
? `EXISTS (
|
||||
SELECT 1 FROM teams
|
||||
WHERE teams.id = "document"."teamId"
|
||||
AND (
|
||||
preferences->> :preference IS NULL
|
||||
OR (preferences->> :preference)::int = :retentionDays
|
||||
)
|
||||
)`
|
||||
: `EXISTS (
|
||||
SELECT 1 FROM teams
|
||||
WHERE teams.id = "document"."teamId"
|
||||
AND (preferences->> :preference)::int = :retentionDays
|
||||
)`
|
||||
),
|
||||
],
|
||||
...this.getPartitionWhereClause("id", partition),
|
||||
},
|
||||
replacements: {
|
||||
preference: TeamPreference.DataRetentionDays,
|
||||
retentionDays,
|
||||
},
|
||||
paranoid: false,
|
||||
limit,
|
||||
});
|
||||
|
||||
if (documents.length > 0) {
|
||||
const countDeletedDocument = await documentPermanentDeleter(documents);
|
||||
Logger.info("task", `Destroyed ${countDeletedDocument} documents`);
|
||||
}
|
||||
}
|
||||
|
||||
public get options() {
|
||||
return {
|
||||
attempts: 1,
|
||||
priority: TaskPriority.Background,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { buildTeam } from "@server/test/factories";
|
||||
import { TeamPreferenceDefaults } from "@shared/constants";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import CleanupPermanentlyDeletedDocumentsTask from "./CleanupPermanentlyDeletedDocumentsTask";
|
||||
import CleanupPermanentlyDeletedDocumentsByRetentionTask from "./CleanupPermanentlyDeletedDocumentsByRetentionTask";
|
||||
|
||||
const props = {
|
||||
limit: 10000,
|
||||
partition: {
|
||||
partitionIndex: 0,
|
||||
partitionCount: 1,
|
||||
},
|
||||
};
|
||||
|
||||
const defaultRetentionDays = TeamPreferenceDefaults[
|
||||
TeamPreference.DataRetentionDays
|
||||
] as number;
|
||||
|
||||
describe("CleanupPermanentlyDeletedDocumentsTask", () => {
|
||||
it("should schedule worker tasks for default and custom retention periods", async () => {
|
||||
const scheduleSpy = jest.spyOn(
|
||||
CleanupPermanentlyDeletedDocumentsByRetentionTask.prototype,
|
||||
"schedule"
|
||||
);
|
||||
|
||||
// Team with custom retention
|
||||
const teamCustom = await buildTeam();
|
||||
const customDays = 7;
|
||||
teamCustom.setPreference(TeamPreference.DataRetentionDays, customDays);
|
||||
await teamCustom.save();
|
||||
|
||||
const task = new CleanupPermanentlyDeletedDocumentsTask();
|
||||
await task.perform(props);
|
||||
|
||||
// Verify that the default retention task was scheduled
|
||||
expect(scheduleSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
retentionDays: defaultRetentionDays,
|
||||
partition: props.partition,
|
||||
})
|
||||
);
|
||||
|
||||
// Verify that the custom retention task was scheduled
|
||||
expect(scheduleSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
retentionDays: customDays,
|
||||
partition: props.partition,
|
||||
})
|
||||
);
|
||||
|
||||
scheduleSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("should always schedule a worker for the default retention period", async () => {
|
||||
const scheduleSpy = jest.spyOn(
|
||||
CleanupPermanentlyDeletedDocumentsByRetentionTask.prototype,
|
||||
"schedule"
|
||||
);
|
||||
|
||||
const task = new CleanupPermanentlyDeletedDocumentsTask();
|
||||
await task.perform(props);
|
||||
|
||||
expect(scheduleSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
retentionDays: defaultRetentionDays,
|
||||
partition: props.partition,
|
||||
})
|
||||
);
|
||||
|
||||
scheduleSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { RetentionPeriodPresets } from "@shared/constants";
|
||||
import { Minute } from "@shared/utils/time";
|
||||
import { TaskPriority } from "./base/BaseTask";
|
||||
import { CronTask, TaskInterval } from "./base/CronTask";
|
||||
import type { Props } from "./base/CronTask";
|
||||
import CleanupPermanentlyDeletedDocumentsByRetentionTask from "./CleanupPermanentlyDeletedDocumentsByRetentionTask";
|
||||
|
||||
export default class CleanupPermanentlyDeletedDocumentsTask extends CronTask {
|
||||
/**
|
||||
* Schedules a worker task for each retention period preset.
|
||||
*
|
||||
* @param props Properties to be used by the task.
|
||||
*/
|
||||
public async perform(props: Props) {
|
||||
const task = new CleanupPermanentlyDeletedDocumentsByRetentionTask();
|
||||
|
||||
for (const days of RetentionPeriodPresets) {
|
||||
if (days === 0) {
|
||||
continue;
|
||||
}
|
||||
await task.schedule({
|
||||
limit: props.limit,
|
||||
retentionDays: days,
|
||||
partition: props.partition,
|
||||
});
|
||||
}
|
||||
|
||||
Logger.debug(
|
||||
"task",
|
||||
`Scheduled ${RetentionPeriodPresets.length - 1} tranches for document cleanup`
|
||||
);
|
||||
}
|
||||
|
||||
public get options() {
|
||||
return {
|
||||
attempts: 1,
|
||||
priority: TaskPriority.Background,
|
||||
};
|
||||
}
|
||||
|
||||
public get cron() {
|
||||
return {
|
||||
interval: TaskInterval.Hour,
|
||||
partitionWindow: 15 * Minute.ms,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
import { Op } from "sequelize";
|
||||
import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
|
||||
import { Document } from "@server/models";
|
||||
import { BaseTask } from "./base/BaseTask";
|
||||
|
||||
@@ -12,18 +11,22 @@ export default class EmptyTrashTask extends BaseTask<Props> {
|
||||
if (!documentIds.length) {
|
||||
return;
|
||||
}
|
||||
const documents = await Document.unscoped().findAll({
|
||||
where: {
|
||||
id: {
|
||||
[Op.in]: documentIds,
|
||||
},
|
||||
// for safety, ensure the documents are in soft-delete state.
|
||||
deletedAt: {
|
||||
[Op.ne]: null,
|
||||
},
|
||||
await Document.unscoped().update(
|
||||
{
|
||||
permanentlyDeletedAt: new Date(),
|
||||
},
|
||||
paranoid: false,
|
||||
});
|
||||
await documentPermanentDeleter(documents);
|
||||
{
|
||||
where: {
|
||||
id: {
|
||||
[Op.in]: documentIds,
|
||||
},
|
||||
// for safety, ensure the documents are in soft-delete state.
|
||||
deletedAt: {
|
||||
[Op.ne]: null,
|
||||
},
|
||||
},
|
||||
paranoid: false,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { subDays } from "date-fns";
|
||||
import { Document } from "@server/models";
|
||||
import { buildDocument, buildTeam } from "@server/test/factories";
|
||||
import { TeamPreferenceDefaults } from "@shared/constants";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import ExpireDocumentsInTrashByRetentionTask from "./ExpireDocumentsInTrashByRetentionTask";
|
||||
|
||||
const props = {
|
||||
partition: {
|
||||
partitionIndex: 0,
|
||||
partitionCount: 1,
|
||||
},
|
||||
};
|
||||
|
||||
const defaultRetentionDays = TeamPreferenceDefaults[
|
||||
TeamPreference.TrashRetentionDays
|
||||
] as number;
|
||||
|
||||
describe("ExpireDocumentsInTrashByRetentionTask", () => {
|
||||
it("should not mark active documents", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
});
|
||||
|
||||
const task = new ExpireDocumentsInTrashByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays: defaultRetentionDays });
|
||||
|
||||
const doc = await Document.unscoped().findOne({
|
||||
where: { teamId: team.id },
|
||||
paranoid: false,
|
||||
});
|
||||
expect(doc?.permanentlyDeletedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("should not mark documents deleted less than 30 days ago (default)", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), defaultRetentionDays - 5),
|
||||
});
|
||||
|
||||
const task = new ExpireDocumentsInTrashByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays: defaultRetentionDays });
|
||||
|
||||
const doc = await Document.unscoped().findOne({
|
||||
where: { teamId: team.id },
|
||||
paranoid: false,
|
||||
});
|
||||
expect(doc?.permanentlyDeletedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("should mark documents deleted more than 30 days ago (default)", async () => {
|
||||
const team = await buildTeam();
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), defaultRetentionDays + 1),
|
||||
});
|
||||
|
||||
const task = new ExpireDocumentsInTrashByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays: defaultRetentionDays });
|
||||
|
||||
const doc = await Document.unscoped().findOne({
|
||||
where: { teamId: team.id },
|
||||
paranoid: false,
|
||||
});
|
||||
expect(doc?.permanentlyDeletedAt).not.toBeNull();
|
||||
});
|
||||
|
||||
it("should respect custom trashRetentionDays", async () => {
|
||||
const retentionDays = 7;
|
||||
const team = await buildTeam();
|
||||
team.setPreference(TeamPreference.TrashRetentionDays, retentionDays);
|
||||
await team.save();
|
||||
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 10),
|
||||
});
|
||||
|
||||
const task = new ExpireDocumentsInTrashByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays });
|
||||
|
||||
const doc = await Document.unscoped().findOne({
|
||||
where: { teamId: team.id },
|
||||
paranoid: false,
|
||||
});
|
||||
expect(doc?.permanentlyDeletedAt).not.toBeNull();
|
||||
});
|
||||
|
||||
it("should not mark documents if within custom trashRetentionDays", async () => {
|
||||
const retentionDays = 90;
|
||||
const team = await buildTeam();
|
||||
team.setPreference(TeamPreference.TrashRetentionDays, retentionDays);
|
||||
await team.save();
|
||||
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
publishedAt: new Date(),
|
||||
deletedAt: subDays(new Date(), 60),
|
||||
});
|
||||
|
||||
const task = new ExpireDocumentsInTrashByRetentionTask();
|
||||
await task.perform({ ...props, retentionDays });
|
||||
|
||||
const doc = await Document.unscoped().findOne({
|
||||
where: { teamId: team.id },
|
||||
paranoid: false,
|
||||
});
|
||||
expect(doc?.permanentlyDeletedAt).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
import { Op, Sequelize } from "sequelize";
|
||||
import { subDays } from "date-fns";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { Document } from "@server/models";
|
||||
import { TeamPreferenceDefaults } from "@shared/constants";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import type { PartitionInfo } from "./base/BaseTask";
|
||||
import { BaseTask, TaskPriority } from "./base/BaseTask";
|
||||
|
||||
export type Props = {
|
||||
/** The trash retention period in days to process in this tranche. */
|
||||
retentionDays: number;
|
||||
/** Partition information for distributing work. */
|
||||
partition?: PartitionInfo;
|
||||
};
|
||||
|
||||
/**
|
||||
* A task that marks documents in the trash for permanent deletion based on a retention period.
|
||||
*/
|
||||
export default class ExpireDocumentsInTrashByRetentionTask extends BaseTask<Props> {
|
||||
public async perform(props: Props) {
|
||||
const { partition, retentionDays } = props;
|
||||
|
||||
// Infinite retention means documents are never expired from trash.
|
||||
if (retentionDays === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const defaultTrashRetentionDays = TeamPreferenceDefaults[
|
||||
TeamPreference.TrashRetentionDays
|
||||
] as number;
|
||||
const isDefault = retentionDays === defaultTrashRetentionDays;
|
||||
|
||||
Logger.debug(
|
||||
"task",
|
||||
`Marking documents past ${retentionDays} day trash timeout as pending permanent deletion…`
|
||||
);
|
||||
|
||||
// Mark documents that have been in the trash for longer than the retention period.
|
||||
// This moves them from "Trash" to "Pending Permanent Deletion" (Retention phase).
|
||||
const [count] = await Document.unscoped().update(
|
||||
{
|
||||
permanentlyDeletedAt: new Date(),
|
||||
},
|
||||
{
|
||||
where: {
|
||||
deletedAt: {
|
||||
[Op.lt]: subDays(new Date(), retentionDays),
|
||||
},
|
||||
permanentlyDeletedAt: {
|
||||
[Op.is]: null,
|
||||
},
|
||||
[Op.and]: [
|
||||
Sequelize.literal(
|
||||
isDefault
|
||||
? `EXISTS (
|
||||
SELECT 1 FROM teams
|
||||
WHERE teams.id = "documents"."teamId"
|
||||
AND (
|
||||
preferences->>'${TeamPreference.TrashRetentionDays}' IS NULL
|
||||
OR (preferences->>'${TeamPreference.TrashRetentionDays}')::int = ${defaultTrashRetentionDays}
|
||||
)
|
||||
)`
|
||||
: `EXISTS (
|
||||
SELECT 1 FROM teams
|
||||
WHERE teams.id = "documents"."teamId"
|
||||
AND (preferences->>'${TeamPreference.TrashRetentionDays}')::int = ${retentionDays}
|
||||
)`
|
||||
),
|
||||
],
|
||||
...this.getPartitionWhereClause("id", partition),
|
||||
},
|
||||
paranoid: false,
|
||||
}
|
||||
);
|
||||
|
||||
if (count > 0) {
|
||||
Logger.info("task", `Marked ${count} documents for permanent deletion`);
|
||||
}
|
||||
}
|
||||
|
||||
public get options() {
|
||||
return {
|
||||
attempts: 1,
|
||||
priority: TaskPriority.Background,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { buildTeam } from "@server/test/factories";
|
||||
import { TeamPreferenceDefaults } from "@shared/constants";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import ExpireDocumentsInTrashTask from "./ExpireDocumentsInTrashTask";
|
||||
import ExpireDocumentsInTrashByRetentionTask from "./ExpireDocumentsInTrashByRetentionTask";
|
||||
|
||||
const props = {
|
||||
limit: 100,
|
||||
partition: {
|
||||
partitionIndex: 0,
|
||||
partitionCount: 1,
|
||||
},
|
||||
};
|
||||
|
||||
const defaultRetentionDays = TeamPreferenceDefaults[
|
||||
TeamPreference.TrashRetentionDays
|
||||
] as number;
|
||||
|
||||
describe("ExpireDocumentsInTrashTask", () => {
|
||||
it("should schedule worker tasks for default and custom retention periods", async () => {
|
||||
const scheduleSpy = jest.spyOn(
|
||||
ExpireDocumentsInTrashByRetentionTask.prototype,
|
||||
"schedule"
|
||||
);
|
||||
|
||||
// Team with custom retention
|
||||
const teamCustom = await buildTeam();
|
||||
const customDays = 7;
|
||||
teamCustom.setPreference(TeamPreference.TrashRetentionDays, customDays);
|
||||
await teamCustom.save();
|
||||
|
||||
const task = new ExpireDocumentsInTrashTask();
|
||||
await task.perform(props);
|
||||
|
||||
// Verify that the default retention task was scheduled
|
||||
expect(scheduleSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
retentionDays: defaultRetentionDays,
|
||||
partition: props.partition,
|
||||
})
|
||||
);
|
||||
|
||||
// Verify that the custom retention task was scheduled.
|
||||
expect(scheduleSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
retentionDays: customDays,
|
||||
partition: props.partition,
|
||||
})
|
||||
);
|
||||
|
||||
scheduleSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("should always schedule a worker for the default retention period", async () => {
|
||||
const scheduleSpy = jest.spyOn(
|
||||
ExpireDocumentsInTrashByRetentionTask.prototype,
|
||||
"schedule"
|
||||
);
|
||||
|
||||
const task = new ExpireDocumentsInTrashTask();
|
||||
await task.perform(props);
|
||||
|
||||
expect(scheduleSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
retentionDays: defaultRetentionDays,
|
||||
partition: props.partition,
|
||||
})
|
||||
);
|
||||
|
||||
scheduleSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { RetentionPeriodPresets } from "@shared/constants";
|
||||
import { Minute } from "@shared/utils/time";
|
||||
import { TaskPriority } from "./base/BaseTask";
|
||||
import { CronTask, TaskInterval } from "./base/CronTask";
|
||||
import type { Props } from "./base/CronTask";
|
||||
import ExpireDocumentsInTrashByRetentionTask from "./ExpireDocumentsInTrashByRetentionTask";
|
||||
|
||||
export default class ExpireDocumentsInTrashTask extends CronTask {
|
||||
/**
|
||||
* Schedules a worker task for each retention period preset.
|
||||
*
|
||||
* @param props Properties to be used by the task.
|
||||
*/
|
||||
public async perform(props: Props) {
|
||||
const task = new ExpireDocumentsInTrashByRetentionTask();
|
||||
|
||||
for (const days of RetentionPeriodPresets) {
|
||||
if (days === 0) {
|
||||
continue;
|
||||
}
|
||||
await task.schedule({
|
||||
retentionDays: days,
|
||||
partition: props.partition,
|
||||
});
|
||||
}
|
||||
|
||||
Logger.debug(
|
||||
"task",
|
||||
`Scheduled ${RetentionPeriodPresets.length - 1} tranches for marking documents for permanent deletion`
|
||||
);
|
||||
}
|
||||
|
||||
public get options() {
|
||||
return {
|
||||
attempts: 1,
|
||||
priority: TaskPriority.Background,
|
||||
};
|
||||
}
|
||||
|
||||
public get cron() {
|
||||
return {
|
||||
interval: TaskInterval.Hour,
|
||||
partitionWindow: 15 * Minute.ms,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -5,8 +5,8 @@ import { QueryTypes } from "sequelize";
|
||||
import { Minute } from "@shared/utils/time";
|
||||
import env from "@server/env";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { TaskPriority } from "./base/BaseTask";
|
||||
import type { PartitionInfo, Props } from "./base/CronTask";
|
||||
import { type PartitionInfo, TaskPriority } from "./base/BaseTask";
|
||||
import type { Props } from "./base/CronTask";
|
||||
import { CronTask, TaskInterval } from "./base/CronTask";
|
||||
import { sequelize, sequelizeReadOnly } from "@server/storage/database";
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import type { Job, JobOptions } from "bull";
|
||||
import { taskQueue } from "../../";
|
||||
import type { WhereOptions } from "sequelize";
|
||||
import { Op } from "sequelize";
|
||||
|
||||
export enum TaskPriority {
|
||||
Background = 40,
|
||||
@@ -8,6 +10,20 @@ export enum TaskPriority {
|
||||
High = 10,
|
||||
}
|
||||
|
||||
/**
|
||||
* Partition information for distributing work across multiple worker instances.
|
||||
*/
|
||||
export type PartitionInfo = {
|
||||
/**
|
||||
* The partition number for this task instance (0-based).
|
||||
*/
|
||||
partitionIndex: number;
|
||||
/**
|
||||
* The total number of partitions.
|
||||
*/
|
||||
partitionCount: number;
|
||||
};
|
||||
|
||||
export abstract class BaseTask<T extends Record<string, any>> {
|
||||
/**
|
||||
* Schedule this task type to be processed asynchronously by a worker.
|
||||
@@ -58,4 +74,105 @@ export abstract class BaseTask<T extends Record<string, any>> {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimized partitioning method for UUID primary keys using range-based distribution.
|
||||
* Divides the UUID space into N equal ranges and assigns each partition a range.
|
||||
*
|
||||
* The UUID space (0x00000000-... to 0xffffffff-...) is divided into N equal ranges.
|
||||
* For example, with 3 partitions:
|
||||
* - Partition 0: '00000000-0000-4000-8000-000000000000' to '55555554-ffff-4fff-bfff-ffffffffffff'
|
||||
* - Partition 1: '55555555-0000-4000-8000-000000000000' to 'aaaaaaaa-ffff-4fff-bfff-ffffffffffff'
|
||||
* - Partition 2: 'aaaaaaab-0000-4000-8000-000000000000' to 'ffffffff-ffff-4fff-bfff-ffffffffffff'
|
||||
*
|
||||
* @param partitionInfo The partition information
|
||||
* @returns The start and end UUID bounds for the partition
|
||||
*/
|
||||
protected getPartitionBounds(
|
||||
partitionInfo: PartitionInfo | undefined
|
||||
): [string, string] {
|
||||
if (!partitionInfo) {
|
||||
return [
|
||||
"00000000-0000-4000-8000-000000000000",
|
||||
"ffffffff-ffff-4fff-bfff-ffffffffffff",
|
||||
];
|
||||
}
|
||||
|
||||
const { partitionIndex, partitionCount } = partitionInfo;
|
||||
|
||||
if (
|
||||
partitionCount <= 0 ||
|
||||
partitionIndex < 0 ||
|
||||
partitionIndex >= partitionCount
|
||||
) {
|
||||
throw new Error(
|
||||
`Invalid partition info: index ${partitionIndex}, count ${partitionCount}`
|
||||
);
|
||||
}
|
||||
|
||||
// 2^32 total possible values for the first 32 bits (4.3 billion)
|
||||
const TOTAL_VALUES = 0x100000000;
|
||||
|
||||
// The maximum possible integer value (0xFFFFFFFF)
|
||||
const MAX_VALUE = TOTAL_VALUES - 1;
|
||||
|
||||
// Ensure even distribution of values by calculating exact range size
|
||||
const rangeSize = Math.floor(TOTAL_VALUES / partitionCount);
|
||||
const rangeStart = partitionIndex * rangeSize;
|
||||
|
||||
let rangeEnd: number;
|
||||
if (partitionIndex === partitionCount - 1) {
|
||||
// The last partition takes any remainder and goes up to the max value
|
||||
rangeEnd = MAX_VALUE;
|
||||
} else {
|
||||
// The end is the start of the *next* partition minus 1
|
||||
rangeEnd = (partitionIndex + 1) * rangeSize - 1;
|
||||
}
|
||||
|
||||
// Use Number.prototype.toString(16) and padStart(8, '0') for the 32-bit hex prefix
|
||||
const startHex = rangeStart.toString(16).padStart(8, "0");
|
||||
const endHex = rangeEnd.toString(16).padStart(8, "0");
|
||||
|
||||
// Start: First 32 bits (prefix) followed by the lowest possible values for the rest
|
||||
// Ensures correct UUID v4 version (4xxx) and variant (8|9|a|bxxx) bits
|
||||
const startUuid = `${startHex}-0000-4000-8000-000000000000`;
|
||||
|
||||
// End: First 32 bits (prefix) followed by the highest possible values for the rest
|
||||
// Ensures correct UUID v4 version (4xxx) and variant (8|9|a|bxxx) bits
|
||||
const endUuid = `${endHex}-ffff-4fff-bfff-ffffffffffff`;
|
||||
|
||||
return [startUuid, endUuid];
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimized partitioning method for UUID primary keys using range-based distribution.
|
||||
* Divides the UUID space into N equal ranges and assigns each partition a range.
|
||||
*
|
||||
* @param idField The name of the UUID primary key field to partition on
|
||||
* @param partitionInfo The partition information
|
||||
* @returns A WHERE clause for partitioned queries
|
||||
*
|
||||
* @example
|
||||
* const where = {
|
||||
* deletedAt: { [Op.lt]: someDate },
|
||||
* ...this.getPartitionWhereClause("id", props.partition)
|
||||
* };
|
||||
*/
|
||||
protected getPartitionWhereClause(
|
||||
idField: string,
|
||||
partitionInfo: PartitionInfo | undefined
|
||||
): WhereOptions {
|
||||
if (!partitionInfo) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const [startUuid, endUuid] = this.getPartitionBounds(partitionInfo);
|
||||
|
||||
return {
|
||||
[idField]: {
|
||||
[Op.gte]: startUuid,
|
||||
[Op.lte]: endUuid,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Op } from "sequelize";
|
||||
import type { PartitionInfo } from "./CronTask";
|
||||
import type { PartitionInfo } from "./BaseTask";
|
||||
import { CronTask, TaskInterval } from "./CronTask";
|
||||
|
||||
// Create a concrete implementation of CronTask for testing
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { WhereOptions } from "sequelize";
|
||||
import { Op } from "sequelize";
|
||||
import type { PartitionInfo } from "./BaseTask";
|
||||
import { BaseTask } from "./BaseTask";
|
||||
|
||||
export enum TaskInterval {
|
||||
@@ -35,20 +34,6 @@ export type TaskSchedule = {
|
||||
partitionWindow?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Partition information for distributing work across multiple worker instances.
|
||||
*/
|
||||
export type PartitionInfo = {
|
||||
/**
|
||||
* The partition number for this task instance (0-based).
|
||||
*/
|
||||
partitionIndex: number;
|
||||
/**
|
||||
* The total number of partitions.
|
||||
*/
|
||||
partitionCount: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Properties for cron-scheduled tasks.
|
||||
*/
|
||||
@@ -57,108 +42,7 @@ export type Props = {
|
||||
partition: PartitionInfo;
|
||||
};
|
||||
|
||||
export abstract class CronTask extends BaseTask<Props> {
|
||||
export abstract class CronTask<P extends Props = Props> extends BaseTask<P> {
|
||||
/** The schedule configuration for this cron task */
|
||||
public abstract get cron(): TaskSchedule;
|
||||
|
||||
/**
|
||||
* Optimized partitioning method for UUID primary keys using range-based distribution.
|
||||
* Divides the UUID space into N equal ranges and assigns each partition a range.
|
||||
*
|
||||
* The UUID space (0x00000000-... to 0xffffffff-...) is divided into N equal ranges.
|
||||
* For example, with 3 partitions:
|
||||
* - Partition 0: '00000000-0000-4000-8000-000000000000' to '55555554-ffff-4fff-bfff-ffffffffffff'
|
||||
* - Partition 1: '55555555-0000-4000-8000-000000000000' to 'aaaaaaaa-ffff-4fff-bfff-ffffffffffff'
|
||||
* - Partition 2: 'aaaaaaab-0000-4000-8000-000000000000' to 'ffffffff-ffff-4fff-bfff-ffffffffffff'
|
||||
*
|
||||
* @param partitionInfo The partition information
|
||||
* @returns The start and end UUID bounds for the partition
|
||||
*/
|
||||
protected getPartitionBounds(
|
||||
partitionInfo: PartitionInfo | undefined
|
||||
): [string, string] {
|
||||
if (!partitionInfo) {
|
||||
return [
|
||||
"00000000-0000-4000-8000-000000000000",
|
||||
"ffffffff-ffff-4fff-bfff-ffffffffffff",
|
||||
];
|
||||
}
|
||||
|
||||
const { partitionIndex, partitionCount } = partitionInfo;
|
||||
|
||||
if (
|
||||
partitionCount <= 0 ||
|
||||
partitionIndex < 0 ||
|
||||
partitionIndex >= partitionCount
|
||||
) {
|
||||
throw new Error(
|
||||
`Invalid partition info: index ${partitionIndex}, count ${partitionCount}`
|
||||
);
|
||||
}
|
||||
|
||||
// 2^32 total possible values for the first 32 bits (4.3 billion)
|
||||
const TOTAL_VALUES = 0x100000000;
|
||||
|
||||
// The maximum possible integer value (0xFFFFFFFF)
|
||||
const MAX_VALUE = TOTAL_VALUES - 1;
|
||||
|
||||
// Ensure even distribution of values by calculating exact range size
|
||||
const rangeSize = Math.floor(TOTAL_VALUES / partitionCount);
|
||||
const rangeStart = partitionIndex * rangeSize;
|
||||
|
||||
let rangeEnd: number;
|
||||
if (partitionIndex === partitionCount - 1) {
|
||||
// The last partition takes any remainder and goes up to the max value
|
||||
rangeEnd = MAX_VALUE;
|
||||
} else {
|
||||
// The end is the start of the *next* partition minus 1
|
||||
rangeEnd = (partitionIndex + 1) * rangeSize - 1;
|
||||
}
|
||||
|
||||
// Use Number.prototype.toString(16) and padStart(8, '0') for the 32-bit hex prefix
|
||||
const startHex = rangeStart.toString(16).padStart(8, "0");
|
||||
const endHex = rangeEnd.toString(16).padStart(8, "0");
|
||||
|
||||
// Start: First 32 bits (prefix) followed by the lowest possible values for the rest
|
||||
// Ensures correct UUID v4 version (4xxx) and variant (8|9|a|bxxx) bits
|
||||
const startUuid = `${startHex}-0000-4000-8000-000000000000`;
|
||||
|
||||
// End: First 32 bits (prefix) followed by the highest possible values for the rest
|
||||
// Ensures correct UUID v4 version (4xxx) and variant (8|9|a|bxxx) bits
|
||||
const endUuid = `${endHex}-ffff-4fff-bfff-ffffffffffff`;
|
||||
|
||||
return [startUuid, endUuid];
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimized partitioning method for UUID primary keys using range-based distribution.
|
||||
* Divides the UUID space into N equal ranges and assigns each partition a range.
|
||||
*
|
||||
* @param idField The name of the UUID primary key field to partition on
|
||||
* @param partitionInfo The partition information
|
||||
* @returns A WHERE clause for partitioned queries
|
||||
*
|
||||
* @example
|
||||
* const where = {
|
||||
* deletedAt: { [Op.lt]: someDate },
|
||||
* ...this.getPartitionWhereClause("id", props.partition)
|
||||
* };
|
||||
*/
|
||||
protected getPartitionWhereClause(
|
||||
idField: string,
|
||||
partitionInfo: PartitionInfo | undefined
|
||||
): WhereOptions {
|
||||
if (!partitionInfo) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const [startUuid, endUuid] = this.getPartitionBounds(partitionInfo);
|
||||
|
||||
return {
|
||||
[idField]: {
|
||||
[Op.gte]: startUuid,
|
||||
[Op.lte]: endUuid,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,6 @@ import documentCreator from "@server/commands/documentCreator";
|
||||
import documentDuplicator from "@server/commands/documentDuplicator";
|
||||
import documentLoader from "@server/commands/documentLoader";
|
||||
import documentMover from "@server/commands/documentMover";
|
||||
import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
|
||||
import documentUpdater from "@server/commands/documentUpdater";
|
||||
import env from "@server/env";
|
||||
import {
|
||||
@@ -405,6 +404,9 @@ router.post(
|
||||
deletedAt: {
|
||||
[Op.ne]: null,
|
||||
},
|
||||
permanentlyDeletedAt: {
|
||||
[Op.is]: null,
|
||||
},
|
||||
[Op.or]: [
|
||||
{
|
||||
collectionId: {
|
||||
@@ -1477,14 +1479,8 @@ router.post(
|
||||
});
|
||||
authorize(user, "permanentDelete", document);
|
||||
|
||||
await documentPermanentDeleter([document]);
|
||||
await Event.createFromContext(ctx, {
|
||||
name: "documents.permanent_delete",
|
||||
documentId: document.id,
|
||||
collectionId: document.collectionId,
|
||||
data: {
|
||||
title: document.title,
|
||||
},
|
||||
await document.updateWithCtx(ctx, {
|
||||
permanentlyDeletedAt: new Date(),
|
||||
});
|
||||
} else {
|
||||
const document = await Document.findByPk(id, {
|
||||
@@ -2097,6 +2093,9 @@ router.post(
|
||||
deletedAt: {
|
||||
[Op.ne]: null,
|
||||
},
|
||||
permanentlyDeletedAt: {
|
||||
[Op.is]: null,
|
||||
},
|
||||
[Op.or]: [
|
||||
{
|
||||
collectionId: {
|
||||
|
||||
@@ -20,6 +20,7 @@ const DocumentsSortParamsSchema = z.object({
|
||||
[
|
||||
"createdAt",
|
||||
"updatedAt",
|
||||
"deletedAt",
|
||||
"publishedAt",
|
||||
"index",
|
||||
"title",
|
||||
|
||||
@@ -3,6 +3,18 @@ import { EmailDisplay, TOCPosition, UserRole } from "@shared/types";
|
||||
import { TeamValidation } from "@shared/validations";
|
||||
import { BaseSchema } from "@server/routes/api/schema";
|
||||
|
||||
const retentionDaysSchema = z
|
||||
.union([
|
||||
z.literal(0),
|
||||
z.literal(7),
|
||||
z.literal(14),
|
||||
z.literal(30),
|
||||
z.literal(90),
|
||||
z.literal(180),
|
||||
z.literal(365),
|
||||
])
|
||||
.optional();
|
||||
|
||||
export const TeamsUpdateSchema = BaseSchema.extend({
|
||||
body: z.object({
|
||||
/** Team name */
|
||||
@@ -66,6 +78,10 @@ export const TeamsUpdateSchema = BaseSchema.extend({
|
||||
emailDisplay: z.enum(EmailDisplay).optional(),
|
||||
/** Whether to prevent shared documents from being embedded in iframes on external websites. */
|
||||
preventDocumentEmbedding: z.boolean().optional(),
|
||||
/** Days to keep documents in trash before retention phase. 0 = infinite. */
|
||||
trashRetentionDays: retentionDaysSchema,
|
||||
/** Days to keep documents in retention phase before permanent deletion. 0 = infinite. */
|
||||
dataRetentionDays: retentionDaysSchema,
|
||||
/** Whether external MCP clients can connect to the workspace. */
|
||||
mcp: z.boolean().optional(),
|
||||
/** List of disabled embed provider titles. */
|
||||
|
||||
+12
-1
@@ -1,4 +1,8 @@
|
||||
import type { TeamPreferences, UserPreferences } from "./types";
|
||||
import type {
|
||||
RetentionPeriodPreset,
|
||||
TeamPreferences,
|
||||
UserPreferences,
|
||||
} from "./types";
|
||||
import {
|
||||
TOCPosition,
|
||||
TeamPreference,
|
||||
@@ -7,6 +11,11 @@ import {
|
||||
NotificationBadgeType,
|
||||
} from "./types";
|
||||
|
||||
/** Allowed retention period values in days. 0 means infinite (never delete). */
|
||||
export const RetentionPeriodPresets: readonly RetentionPeriodPreset[] = [
|
||||
0, 7, 14, 30, 90, 180, 365,
|
||||
];
|
||||
|
||||
export const MAX_AVATAR_DISPLAY = 6;
|
||||
|
||||
export const Pagination = {
|
||||
@@ -34,6 +43,8 @@ export const TeamPreferenceDefaults: TeamPreferences = {
|
||||
[TeamPreference.CustomTheme]: undefined,
|
||||
[TeamPreference.TocPosition]: TOCPosition.Left,
|
||||
[TeamPreference.PreventDocumentEmbedding]: false,
|
||||
[TeamPreference.TrashRetentionDays]: 30,
|
||||
[TeamPreference.DataRetentionDays]: 30,
|
||||
[TeamPreference.EmailDisplay]: EmailDisplay.Members,
|
||||
[TeamPreference.MCP]: true,
|
||||
[TeamPreference.DisabledEmbeds]: [],
|
||||
|
||||
@@ -272,6 +272,8 @@
|
||||
"New": "New",
|
||||
"Only visible to you": "Only visible to you",
|
||||
"Draft": "Draft",
|
||||
"Permanently deletes in {{ days }} days": "Permanently deletes in {{ days }} days",
|
||||
"{{ days }} days": "{{ days }} days",
|
||||
"You updated": "You updated",
|
||||
"{{ userName }} updated": "{{ userName }} updated",
|
||||
"You deleted": "You deleted",
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
/** Allowed retention period in days. 0 means infinite (never delete). */
|
||||
export type RetentionPeriodPreset = 0 | 7 | 14 | 30 | 90 | 180 | 365;
|
||||
|
||||
/** Available user roles. */
|
||||
export enum UserRole {
|
||||
Admin = "admin",
|
||||
@@ -406,6 +409,10 @@ export enum TeamPreference {
|
||||
TocPosition = "tocPosition",
|
||||
/** Whether to prevent shared documents from being embedded in iframes on external websites. */
|
||||
PreventDocumentEmbedding = "preventDocumentEmbedding",
|
||||
/** The number of days to keep documents in the trash before moving to the retention phase. */
|
||||
TrashRetentionDays = "trashRetentionDays",
|
||||
/** The number of days to keep documents in the retention phase before permanent deletion. */
|
||||
DataRetentionDays = "dataRetentionDays",
|
||||
/** Who can see user email addresses. */
|
||||
EmailDisplay = "emailDisplay",
|
||||
/** Whether external MCP clients can connect to the workspace. */
|
||||
@@ -426,6 +433,8 @@ export type TeamPreferences = {
|
||||
[TeamPreference.CustomTheme]?: Partial<CustomTheme>;
|
||||
[TeamPreference.TocPosition]?: TOCPosition;
|
||||
[TeamPreference.PreventDocumentEmbedding]?: boolean;
|
||||
[TeamPreference.TrashRetentionDays]?: RetentionPeriodPreset;
|
||||
[TeamPreference.DataRetentionDays]?: RetentionPeriodPreset;
|
||||
[TeamPreference.EmailDisplay]?: EmailDisplay;
|
||||
[TeamPreference.MCP]?: boolean;
|
||||
[TeamPreference.DisabledEmbeds]?: string[];
|
||||
|
||||
Reference in New Issue
Block a user