mirror of
https://github.com/outline/outline.git
synced 2026-06-13 03:14:59 +03:00
feat: Extract search into pluggable provider system (#11448)
* feat: Extract search into pluggable provider system Refactors the monolithic SearchHelper into a pluggable search provider architecture, enabling alternative search backends (Elasticsearch, Turbopuffer, etc.) while preserving PostgreSQL full-text search as the default. The SEARCH_PROVIDER env var selects the active provider. - Add BaseSearchProvider abstract class and SearchProviderManager - Add Hook.SearchProvider to the plugin system - Move PostgreSQL search logic into plugins/postgres-search/ - Add SearchIndexProcessor for event-driven index sync - Update all callers to use the provider manager directly - Keep SearchHelper as a deprecated thin wrapper for backwards compat Closes #11347 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * refactor: Remove deprecated SearchHelper wrapper All callers now use SearchProviderManager directly, so the thin delegation wrapper is no longer needed. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * refactor: Rename postgres-search plugin to search-postgres Renames the plugin folder and id so that future search provider plugins (e.g. search-elasticsearch, search-turbopuffer) will be colocated alphabetically. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * refactor: Remove special-case plugin import from SearchProviderManager Make PluginManager.loadPlugins resilient to individual plugin load failures so SearchProviderManager can use the standard getHooks path without needing to directly import the search-postgres plugin. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * test: Add missing search provider tests for full coverage parity Adds all tests that existed in the old SearchHelper.test.ts but were missing from PostgresSearchProvider.test.ts, including searchTitlesForUser status filters, collection filtering, group memberships, and sorting tests. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feedback --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -24,7 +24,11 @@ class UnfurlsStore extends Store<Unfurl<any>> {
|
||||
}): Promise<Unfurl<UnfurlType> | undefined> => {
|
||||
try {
|
||||
const protocol = new URL(url).protocol;
|
||||
if (protocol !== "http:" && protocol !== "https:" && protocol !== "mention:") {
|
||||
if (
|
||||
protocol !== "http:" &&
|
||||
protocol !== "https:" &&
|
||||
protocol !== "mention:"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
} catch (_err) {
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"id": "search-postgres",
|
||||
"name": "PostgreSQL Search",
|
||||
"priority": 0,
|
||||
"description": "Full-text search powered by PostgreSQL tsvector."
|
||||
}
|
||||
+71
-68
@@ -4,7 +4,6 @@ import {
|
||||
SortFilter,
|
||||
StatusFilter,
|
||||
} from "@shared/types";
|
||||
import SearchHelper from "@server/models/helpers/SearchHelper";
|
||||
import {
|
||||
buildDocument,
|
||||
buildDraftDocument,
|
||||
@@ -14,15 +13,19 @@ import {
|
||||
buildShare,
|
||||
buildGroup,
|
||||
} from "@server/test/factories";
|
||||
import UserMembership from "../UserMembership";
|
||||
import GroupMembership from "../GroupMembership";
|
||||
import UserMembership from "@server/models/UserMembership";
|
||||
import GroupMembership from "@server/models/GroupMembership";
|
||||
import SearchProviderManager from "@server/utils/SearchProviderManager";
|
||||
import PostgresSearchProvider from "./PostgresSearchProvider";
|
||||
|
||||
const provider = SearchProviderManager.getProvider();
|
||||
|
||||
beforeEach(async () => {
|
||||
jest.resetAllMocks();
|
||||
await buildDocument();
|
||||
});
|
||||
|
||||
describe("SearchHelper", () => {
|
||||
describe("PostgresSearchProvider", () => {
|
||||
describe("#searchForTeam", () => {
|
||||
it("should return search results from public collections", async () => {
|
||||
const team = await buildTeam();
|
||||
@@ -34,7 +37,7 @@ describe("SearchHelper", () => {
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForTeam(team, {
|
||||
const { results } = await provider.searchForTeam(team, {
|
||||
query: "test",
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
@@ -58,7 +61,7 @@ describe("SearchHelper", () => {
|
||||
title: "document 2",
|
||||
}),
|
||||
]);
|
||||
const { results } = await SearchHelper.searchForTeam(team);
|
||||
const { results } = await provider.searchForTeam(team);
|
||||
expect(results.length).toBe(2);
|
||||
expect(results.map((r) => r.document.id).sort()).toEqual(
|
||||
documents.map((doc) => doc.id).sort()
|
||||
@@ -76,7 +79,7 @@ describe("SearchHelper", () => {
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForTeam(team, {
|
||||
const { results } = await provider.searchForTeam(team, {
|
||||
query: "test",
|
||||
});
|
||||
expect(results.length).toBe(0);
|
||||
@@ -93,7 +96,7 @@ describe("SearchHelper", () => {
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForTeam(team, {
|
||||
const { results } = await provider.searchForTeam(team, {
|
||||
query: "test",
|
||||
collectionId: collection.id,
|
||||
});
|
||||
@@ -122,7 +125,7 @@ describe("SearchHelper", () => {
|
||||
includeChildDocuments: true,
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForTeam(team, {
|
||||
const { results } = await provider.searchForTeam(team, {
|
||||
query: "test",
|
||||
collectionId: collection.id,
|
||||
share,
|
||||
@@ -132,7 +135,7 @@ describe("SearchHelper", () => {
|
||||
|
||||
it("should handle no collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const { results } = await SearchHelper.searchForTeam(team, {
|
||||
const { results } = await provider.searchForTeam(team, {
|
||||
query: "test",
|
||||
});
|
||||
expect(results.length).toBe(0);
|
||||
@@ -148,7 +151,7 @@ describe("SearchHelper", () => {
|
||||
collectionId: collection.id,
|
||||
title: "test with backslash \\",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForTeam(team, {
|
||||
const { results } = await provider.searchForTeam(team, {
|
||||
query: "test with backslash \\",
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
@@ -170,7 +173,7 @@ describe("SearchHelper", () => {
|
||||
collectionId: collection.id,
|
||||
title: "test number 2",
|
||||
});
|
||||
const { total } = await SearchHelper.searchForTeam(team, {
|
||||
const { total } = await provider.searchForTeam(team, {
|
||||
query: "test",
|
||||
});
|
||||
expect(total).toBe(2);
|
||||
@@ -188,7 +191,7 @@ describe("SearchHelper", () => {
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { total } = await SearchHelper.searchForTeam(team, {
|
||||
const { total } = await provider.searchForTeam(team, {
|
||||
query: "test number",
|
||||
});
|
||||
expect(total).toBe(1);
|
||||
@@ -206,7 +209,7 @@ describe("SearchHelper", () => {
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { total } = await SearchHelper.searchForTeam(team, {
|
||||
const { total } = await provider.searchForTeam(team, {
|
||||
query: "title doesn't exist",
|
||||
});
|
||||
expect(total).toBe(0);
|
||||
@@ -234,7 +237,7 @@ describe("SearchHelper", () => {
|
||||
deletedAt: new Date(),
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "test",
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
@@ -263,7 +266,7 @@ describe("SearchHelper", () => {
|
||||
title: "document 2",
|
||||
}),
|
||||
]);
|
||||
const { results } = await SearchHelper.searchForUser(user);
|
||||
const { results } = await provider.searchForUser(user);
|
||||
expect(results.length).toBe(2);
|
||||
expect(results.map((r) => r.document.id).sort()).toEqual(
|
||||
documents.map((doc) => doc.id).sort()
|
||||
@@ -291,7 +294,7 @@ describe("SearchHelper", () => {
|
||||
title: "document 2",
|
||||
}),
|
||||
]);
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
collectionId: collection.id,
|
||||
});
|
||||
expect(results.length).toBe(2);
|
||||
@@ -339,7 +342,7 @@ describe("SearchHelper", () => {
|
||||
title: "document 2 in collection 2",
|
||||
}),
|
||||
]);
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
collectionId: collection1.id,
|
||||
});
|
||||
expect(results.length).toBe(2);
|
||||
@@ -351,7 +354,7 @@ describe("SearchHelper", () => {
|
||||
it("should handle no collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({ teamId: team.id });
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "test",
|
||||
});
|
||||
expect(results.length).toBe(0);
|
||||
@@ -381,7 +384,7 @@ describe("SearchHelper", () => {
|
||||
title: "test",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Draft],
|
||||
});
|
||||
@@ -406,7 +409,7 @@ describe("SearchHelper", () => {
|
||||
permission: DocumentPermission.Read,
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Published, StatusFilter.Archived],
|
||||
});
|
||||
@@ -437,7 +440,7 @@ describe("SearchHelper", () => {
|
||||
title: "test",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Published],
|
||||
});
|
||||
@@ -474,7 +477,7 @@ describe("SearchHelper", () => {
|
||||
title: "test",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Archived],
|
||||
});
|
||||
@@ -502,7 +505,7 @@ describe("SearchHelper", () => {
|
||||
title: "test",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Archived, StatusFilter.Published],
|
||||
});
|
||||
@@ -530,7 +533,7 @@ describe("SearchHelper", () => {
|
||||
title: "archived not draft",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "draft",
|
||||
statusFilter: [StatusFilter.Published, StatusFilter.Draft],
|
||||
});
|
||||
@@ -558,7 +561,7 @@ describe("SearchHelper", () => {
|
||||
title: "archived not draft",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "draft",
|
||||
statusFilter: [StatusFilter.Draft, StatusFilter.Archived],
|
||||
});
|
||||
@@ -584,7 +587,7 @@ describe("SearchHelper", () => {
|
||||
collectionId: collection.id,
|
||||
title: "test number 2",
|
||||
});
|
||||
const { total } = await SearchHelper.searchForUser(user, {
|
||||
const { total } = await provider.searchForUser(user, {
|
||||
query: "test",
|
||||
});
|
||||
expect(total).toBe(2);
|
||||
@@ -605,7 +608,7 @@ describe("SearchHelper", () => {
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { total } = await SearchHelper.searchForUser(user, {
|
||||
const { total } = await provider.searchForUser(user, {
|
||||
query: "test number",
|
||||
});
|
||||
expect(total).toBe(1);
|
||||
@@ -626,7 +629,7 @@ describe("SearchHelper", () => {
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { total } = await SearchHelper.searchForUser(user, {
|
||||
const { total } = await provider.searchForUser(user, {
|
||||
query: "title doesn't exist",
|
||||
});
|
||||
expect(total).toBe(0);
|
||||
@@ -647,7 +650,7 @@ describe("SearchHelper", () => {
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { total } = await SearchHelper.searchForUser(user, {
|
||||
const { total } = await provider.searchForUser(user, {
|
||||
query: `"test number"`,
|
||||
});
|
||||
expect(total).toBe(1);
|
||||
@@ -668,7 +671,7 @@ describe("SearchHelper", () => {
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { total } = await SearchHelper.searchForUser(user, {
|
||||
const { total } = await provider.searchForUser(user, {
|
||||
query: "env: ",
|
||||
});
|
||||
expect(total).toBe(1);
|
||||
@@ -681,7 +684,7 @@ describe("SearchHelper", () => {
|
||||
const collection = await buildCollection({
|
||||
userId: otherUser.id,
|
||||
teamId: team.id,
|
||||
permission: null, // private collection
|
||||
permission: null,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
userId: otherUser.id,
|
||||
@@ -690,7 +693,6 @@ describe("SearchHelper", () => {
|
||||
title: "group test document",
|
||||
});
|
||||
|
||||
// Document with no access should not appear in results
|
||||
await buildDocument({
|
||||
userId: otherUser.id,
|
||||
teamId: team.id,
|
||||
@@ -698,7 +700,6 @@ describe("SearchHelper", () => {
|
||||
title: "group test document 2",
|
||||
});
|
||||
|
||||
// Create a group and add the user to it
|
||||
const group = await buildGroup({
|
||||
teamId: team.id,
|
||||
});
|
||||
@@ -708,14 +709,13 @@ describe("SearchHelper", () => {
|
||||
},
|
||||
});
|
||||
|
||||
// Add group membership to the document
|
||||
await GroupMembership.create({
|
||||
createdById: otherUser.id,
|
||||
groupId: group.id,
|
||||
documentId: document.id,
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "group test",
|
||||
});
|
||||
|
||||
@@ -739,7 +739,7 @@ describe("SearchHelper", () => {
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "test",
|
||||
});
|
||||
expect(documents.length).toBe(1);
|
||||
@@ -774,7 +774,7 @@ describe("SearchHelper", () => {
|
||||
collectionId: collection1.id,
|
||||
title: "test",
|
||||
});
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "test",
|
||||
collectionId: collection.id,
|
||||
});
|
||||
@@ -785,7 +785,7 @@ describe("SearchHelper", () => {
|
||||
it("should handle no collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({ teamId: team.id });
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "test",
|
||||
});
|
||||
expect(documents.length).toBe(0);
|
||||
@@ -815,7 +815,7 @@ describe("SearchHelper", () => {
|
||||
title: "test",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Draft],
|
||||
});
|
||||
@@ -846,7 +846,7 @@ describe("SearchHelper", () => {
|
||||
title: "test",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Published],
|
||||
});
|
||||
@@ -883,7 +883,7 @@ describe("SearchHelper", () => {
|
||||
title: "test",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Archived],
|
||||
});
|
||||
@@ -911,7 +911,7 @@ describe("SearchHelper", () => {
|
||||
title: "test",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "test",
|
||||
statusFilter: [StatusFilter.Archived, StatusFilter.Published],
|
||||
});
|
||||
@@ -939,7 +939,7 @@ describe("SearchHelper", () => {
|
||||
title: "archived not draft",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "draft",
|
||||
statusFilter: [StatusFilter.Published, StatusFilter.Draft],
|
||||
});
|
||||
@@ -967,7 +967,7 @@ describe("SearchHelper", () => {
|
||||
title: "archived not draft",
|
||||
archivedAt: new Date(),
|
||||
});
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "draft",
|
||||
statusFilter: [StatusFilter.Draft, StatusFilter.Archived],
|
||||
});
|
||||
@@ -981,7 +981,7 @@ describe("SearchHelper", () => {
|
||||
const collection = await buildCollection({
|
||||
userId: otherUser.id,
|
||||
teamId: team.id,
|
||||
permission: null, // private collection
|
||||
permission: null,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
userId: otherUser.id,
|
||||
@@ -990,7 +990,6 @@ describe("SearchHelper", () => {
|
||||
title: "group title test document",
|
||||
});
|
||||
|
||||
// Document with no access should not appear in results
|
||||
await buildDocument({
|
||||
userId: otherUser.id,
|
||||
teamId: team.id,
|
||||
@@ -998,7 +997,6 @@ describe("SearchHelper", () => {
|
||||
title: "group title test document 2",
|
||||
});
|
||||
|
||||
// Create a group and add the user to it
|
||||
const group = await buildGroup({
|
||||
teamId: team.id,
|
||||
});
|
||||
@@ -1008,14 +1006,13 @@ describe("SearchHelper", () => {
|
||||
},
|
||||
});
|
||||
|
||||
// Add group membership to the document
|
||||
await GroupMembership.create({
|
||||
createdById: otherUser.id,
|
||||
groupId: group.id,
|
||||
documentId: document.id,
|
||||
});
|
||||
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
const documents = await provider.searchTitlesForUser(user, {
|
||||
query: "group title",
|
||||
});
|
||||
|
||||
@@ -1039,7 +1036,7 @@ describe("SearchHelper", () => {
|
||||
name: "Other Collection",
|
||||
});
|
||||
|
||||
const results = await SearchHelper.searchCollectionsForUser(user, {
|
||||
const results = await provider.searchCollectionsForUser(user, {
|
||||
query: "test",
|
||||
});
|
||||
|
||||
@@ -1061,7 +1058,7 @@ describe("SearchHelper", () => {
|
||||
name: "Beta",
|
||||
});
|
||||
|
||||
const results = await SearchHelper.searchCollectionsForUser(user);
|
||||
const results = await provider.searchCollectionsForUser(user);
|
||||
|
||||
expect(results.length).toBe(2);
|
||||
expect(results[0].id).toBe(collection1.id);
|
||||
@@ -1096,7 +1093,7 @@ describe("SearchHelper", () => {
|
||||
title: "Beta Document",
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
sort: SortFilter.Title,
|
||||
direction: DirectionFilter.ASC,
|
||||
});
|
||||
@@ -1133,7 +1130,7 @@ describe("SearchHelper", () => {
|
||||
title: "Beta Document",
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
sort: SortFilter.Title,
|
||||
direction: DirectionFilter.DESC,
|
||||
});
|
||||
@@ -1176,7 +1173,7 @@ describe("SearchHelper", () => {
|
||||
updatedAt: new Date("2023-12-01"),
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
sort: SortFilter.CreatedAt,
|
||||
direction: DirectionFilter.ASC,
|
||||
});
|
||||
@@ -1216,7 +1213,7 @@ describe("SearchHelper", () => {
|
||||
updatedAt: new Date("2023-06-01"),
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user);
|
||||
const { results } = await provider.searchForUser(user);
|
||||
|
||||
expect(results.length).toBe(3);
|
||||
expect(results[0].document.id).toBe(doc2.id);
|
||||
@@ -1252,7 +1249,7 @@ describe("SearchHelper", () => {
|
||||
updatedAt: new Date("2023-01-01"),
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "search",
|
||||
});
|
||||
|
||||
@@ -1288,7 +1285,7 @@ describe("SearchHelper", () => {
|
||||
updatedAt: new Date("2025-12-01"),
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await provider.searchForUser(user, {
|
||||
query: "search",
|
||||
sort: SortFilter.UpdatedAt,
|
||||
direction: DirectionFilter.DESC,
|
||||
@@ -1326,7 +1323,7 @@ describe("SearchHelper", () => {
|
||||
});
|
||||
|
||||
// Without popularity boost, pure relevance should win
|
||||
const { results: withoutBoost } = await SearchHelper.searchForTeam(team, {
|
||||
const { results: withoutBoost } = await provider.searchForTeam(team, {
|
||||
query: "testing",
|
||||
usePopularityBoost: false,
|
||||
});
|
||||
@@ -1335,7 +1332,7 @@ describe("SearchHelper", () => {
|
||||
expect(withoutBoost[0].document.id).toBe(relevantDoc.id);
|
||||
|
||||
// With popularity boost, the popular document may rank higher
|
||||
const { results: withBoost } = await SearchHelper.searchForTeam(team, {
|
||||
const { results: withBoost } = await provider.searchForTeam(team, {
|
||||
query: "testing",
|
||||
usePopularityBoost: true,
|
||||
});
|
||||
@@ -1350,22 +1347,28 @@ describe("SearchHelper", () => {
|
||||
|
||||
describe("webSearchQuery", () => {
|
||||
it("should correctly sanitize query", () => {
|
||||
expect(SearchHelper.webSearchQuery("one/two")).toBe("one/two:*");
|
||||
expect(SearchHelper.webSearchQuery("one\\two")).toBe("one\\\\two:*");
|
||||
expect(SearchHelper.webSearchQuery("test''")).toBe("test");
|
||||
expect(PostgresSearchProvider.webSearchQuery("one/two")).toBe(
|
||||
"one/two:*"
|
||||
);
|
||||
expect(PostgresSearchProvider.webSearchQuery("one\\two")).toBe(
|
||||
"one\\\\two:*"
|
||||
);
|
||||
expect(PostgresSearchProvider.webSearchQuery("test''")).toBe("test");
|
||||
});
|
||||
it("should wildcard unquoted queries", () => {
|
||||
expect(SearchHelper.webSearchQuery("test")).toBe("test:*");
|
||||
expect(SearchHelper.webSearchQuery("'")).toBe("");
|
||||
expect(SearchHelper.webSearchQuery("'quoted'")).toBe(`"quoted":*`);
|
||||
expect(PostgresSearchProvider.webSearchQuery("test")).toBe("test:*");
|
||||
expect(PostgresSearchProvider.webSearchQuery("'")).toBe("");
|
||||
expect(PostgresSearchProvider.webSearchQuery("'quoted'")).toBe(
|
||||
`"quoted":*`
|
||||
);
|
||||
});
|
||||
it("should wildcard multi-word queries", () => {
|
||||
expect(SearchHelper.webSearchQuery("this is a test")).toBe(
|
||||
expect(PostgresSearchProvider.webSearchQuery("this is a test")).toBe(
|
||||
"this&is&a&test:*"
|
||||
);
|
||||
});
|
||||
it("should not wildcard quoted queries", () => {
|
||||
expect(SearchHelper.webSearchQuery(`"this is a test"`)).toBe(
|
||||
expect(PostgresSearchProvider.webSearchQuery(`"this is a test"`)).toBe(
|
||||
`"this<->is<->a<->test"`
|
||||
);
|
||||
});
|
||||
+102
-84
@@ -11,63 +11,23 @@ import type {
|
||||
WhereOptions,
|
||||
} from "sequelize";
|
||||
import { Op, Sequelize } from "sequelize";
|
||||
import type { DateFilter } from "@shared/types";
|
||||
import { DirectionFilter, SortFilter } from "@shared/types";
|
||||
import { StatusFilter } from "@shared/types";
|
||||
import type { SearchableModel } from "@shared/types";
|
||||
import { DirectionFilter, SortFilter, StatusFilter } from "@shared/types";
|
||||
import { regexIndexOf, regexLastIndexOf } from "@shared/utils/string";
|
||||
import { getUrls } from "@shared/utils/urls";
|
||||
import { ValidationError } from "@server/errors";
|
||||
import Collection from "@server/models/Collection";
|
||||
import type Comment from "@server/models/Comment";
|
||||
import Document from "@server/models/Document";
|
||||
import type Share from "@server/models/Share";
|
||||
import Team from "@server/models/Team";
|
||||
import User from "@server/models/User";
|
||||
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
|
||||
import { sequelize } from "@server/storage/database";
|
||||
import { DocumentHelper } from "./DocumentHelper";
|
||||
|
||||
type SearchResponse = {
|
||||
results: {
|
||||
/** The search ranking, for sorting results */
|
||||
ranking: number;
|
||||
/** A snippet of contextual text around the search result */
|
||||
context?: string;
|
||||
/** The document result */
|
||||
document: Document;
|
||||
}[];
|
||||
/** The total number of results for the search query without pagination */
|
||||
total: number;
|
||||
};
|
||||
|
||||
type SearchOptions = {
|
||||
/** The query limit for pagination */
|
||||
limit?: number;
|
||||
/** The query offset for pagination */
|
||||
offset?: number;
|
||||
/** The text to search for */
|
||||
query?: string;
|
||||
/** Limit results to a collection. Authorization is presumed to have been done before passing to this helper. */
|
||||
collectionId?: string | null;
|
||||
/** Limit results to a shared document. */
|
||||
share?: Share;
|
||||
/** Limit results to a date range. */
|
||||
dateFilter?: DateFilter;
|
||||
/** Status of the documents to return */
|
||||
statusFilter?: StatusFilter[];
|
||||
/** Limit results to a list of documents. */
|
||||
documentIds?: string[];
|
||||
/** Limit results to a list of users that collaborated on the document. */
|
||||
collaboratorIds?: string[];
|
||||
/** The minimum number of words to be returned in the contextual snippet */
|
||||
snippetMinWords?: number;
|
||||
/** The maximum number of words to be returned in the contextual snippet */
|
||||
snippetMaxWords?: number;
|
||||
/** The field to sort results by */
|
||||
sort?: SortFilter;
|
||||
/** The sort direction */
|
||||
direction?: DirectionFilter;
|
||||
/** Whether to boost results by popularity score. Defaults to true. */
|
||||
usePopularityBoost?: boolean;
|
||||
};
|
||||
import type {
|
||||
SearchOptions,
|
||||
SearchResponse,
|
||||
} from "@server/utils/BaseSearchProvider";
|
||||
import { BaseSearchProvider } from "@server/utils/BaseSearchProvider";
|
||||
|
||||
type RankedDocument = Document & {
|
||||
id: string;
|
||||
@@ -76,24 +36,31 @@ type RankedDocument = Document & {
|
||||
};
|
||||
};
|
||||
|
||||
export default class SearchHelper {
|
||||
/**
|
||||
* Search provider that uses PostgreSQL full-text search via tsvector.
|
||||
* Indexing is handled by database triggers, so index/remove/updateMetadata
|
||||
* are no-ops.
|
||||
*/
|
||||
export default class PostgresSearchProvider extends BaseSearchProvider {
|
||||
id = "postgres";
|
||||
|
||||
/**
|
||||
* The maximum length of a search query.
|
||||
*/
|
||||
public static maxQueryLength = 1000;
|
||||
|
||||
/**
|
||||
* Cached regex pattern for single quotes to avoid recompilation
|
||||
* Cached regex pattern for single quotes to avoid recompilation.
|
||||
*/
|
||||
private static readonly SINGLE_QUOTE_REGEX = /'+/g;
|
||||
|
||||
/**
|
||||
* Cached regex pattern for quoted queries
|
||||
* Cached regex pattern for quoted queries.
|
||||
*/
|
||||
private static readonly QUOTED_QUERY_REGEX = /"([^"]*)"/g;
|
||||
|
||||
/**
|
||||
* Cached regex pattern for break characters
|
||||
* Cached regex pattern for break characters.
|
||||
*/
|
||||
private static readonly BREAK_CHARS_REGEX = new RegExp(
|
||||
`[ .,"'\n。!?!?…]`,
|
||||
@@ -101,7 +68,7 @@ export default class SearchHelper {
|
||||
);
|
||||
|
||||
/**
|
||||
* Cached stop words set for efficient lookup
|
||||
* Cached stop words set for efficient lookup.
|
||||
* Based on: https://github.com/postgres/postgres/blob/fc0d0ce978752493868496be6558fa17b7c4c3cf/src/backend/snowball/stopwords/english.stop
|
||||
*/
|
||||
private static readonly STOP_WORDS = new Set([
|
||||
@@ -215,13 +182,13 @@ export default class SearchHelper {
|
||||
"should",
|
||||
]);
|
||||
|
||||
public static async searchForTeam(
|
||||
async searchForTeam(
|
||||
team: Team,
|
||||
options: SearchOptions = {}
|
||||
): Promise<SearchResponse> {
|
||||
const { limit = 15, offset = 0, query } = options;
|
||||
|
||||
const where = await this.buildWhere(team, {
|
||||
const where = await PostgresSearchProvider.buildWhere(team, {
|
||||
...options,
|
||||
statusFilter: [...(options.statusFilter || []), StatusFilter.Published],
|
||||
});
|
||||
@@ -256,7 +223,7 @@ export default class SearchHelper {
|
||||
});
|
||||
}
|
||||
|
||||
const findOptions = this.buildFindOptions({
|
||||
const findOptions = PostgresSearchProvider.buildFindOptions({
|
||||
query,
|
||||
sort: options.sort,
|
||||
direction: options.direction,
|
||||
@@ -292,7 +259,7 @@ export default class SearchHelper {
|
||||
],
|
||||
});
|
||||
|
||||
return this.buildResponse({
|
||||
return PostgresSearchProvider.buildResponse({
|
||||
query,
|
||||
results,
|
||||
documents,
|
||||
@@ -306,12 +273,12 @@ export default class SearchHelper {
|
||||
}
|
||||
}
|
||||
|
||||
public static async searchTitlesForUser(
|
||||
async searchTitlesForUser(
|
||||
user: User,
|
||||
options: SearchOptions = {}
|
||||
): Promise<Document[]> {
|
||||
const { limit = 15, offset = 0, query, ...rest } = options;
|
||||
const where = await this.buildWhere(user, rest);
|
||||
const where = await PostgresSearchProvider.buildWhere(user, rest);
|
||||
|
||||
if (query) {
|
||||
where[Op.and].push({
|
||||
@@ -379,7 +346,7 @@ export default class SearchHelper {
|
||||
});
|
||||
}
|
||||
|
||||
public static async searchCollectionsForUser(
|
||||
async searchCollectionsForUser(
|
||||
user: User,
|
||||
options: SearchOptions = {}
|
||||
): Promise<Collection[]> {
|
||||
@@ -408,15 +375,15 @@ export default class SearchHelper {
|
||||
});
|
||||
}
|
||||
|
||||
public static async searchForUser(
|
||||
async searchForUser(
|
||||
user: User,
|
||||
options: SearchOptions = {}
|
||||
): Promise<SearchResponse> {
|
||||
const { limit = 15, offset = 0, query } = options;
|
||||
|
||||
const where = await this.buildWhere(user, options);
|
||||
const where = await PostgresSearchProvider.buildWhere(user, options);
|
||||
|
||||
const findOptions = this.buildFindOptions({
|
||||
const findOptions = PostgresSearchProvider.buildFindOptions({
|
||||
query,
|
||||
sort: options.sort,
|
||||
direction: options.direction,
|
||||
@@ -484,7 +451,7 @@ export default class SearchHelper {
|
||||
: countQuery,
|
||||
]);
|
||||
|
||||
return this.buildResponse({
|
||||
return PostgresSearchProvider.buildResponse({
|
||||
query,
|
||||
results,
|
||||
documents,
|
||||
@@ -498,6 +465,49 @@ export default class SearchHelper {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* No-op for PostgreSQL — indexing is handled by database triggers.
|
||||
*
|
||||
* @param _model - unused.
|
||||
* @param _item - unused.
|
||||
*/
|
||||
async index(
|
||||
_model: SearchableModel,
|
||||
_item: Document | Collection | Comment
|
||||
): Promise<void> {
|
||||
// PostgreSQL uses tsvector triggers for indexing
|
||||
}
|
||||
|
||||
/**
|
||||
* No-op for PostgreSQL — removal is handled by database cascades.
|
||||
*
|
||||
* @param _model - unused.
|
||||
* @param _id - unused.
|
||||
* @param _teamId - unused.
|
||||
*/
|
||||
async remove(
|
||||
_model: SearchableModel,
|
||||
_id: string,
|
||||
_teamId: string
|
||||
): Promise<void> {
|
||||
// PostgreSQL handles removal via cascading deletes
|
||||
}
|
||||
|
||||
/**
|
||||
* No-op for PostgreSQL — metadata is stored in the same tables.
|
||||
*
|
||||
* @param _model - unused.
|
||||
* @param _id - unused.
|
||||
* @param _metadata - unused.
|
||||
*/
|
||||
async updateMetadata(
|
||||
_model: SearchableModel,
|
||||
_id: string,
|
||||
_metadata: Record<string, unknown>
|
||||
): Promise<void> {
|
||||
// PostgreSQL metadata lives in the same row as the document
|
||||
}
|
||||
|
||||
private static buildFindOptions({
|
||||
query,
|
||||
sort,
|
||||
@@ -519,7 +529,7 @@ export default class SearchHelper {
|
||||
: `ts_rank("searchVector", to_tsquery('english', :query))`;
|
||||
|
||||
attributes.push([Sequelize.literal(rankExpression), "searchRanking"]);
|
||||
replacements["query"] = this.webSearchQuery(query);
|
||||
replacements["query"] = PostgresSearchProvider.webSearchQuery(query);
|
||||
}
|
||||
|
||||
// When searching with a query and no explicit sort, prioritize search
|
||||
@@ -551,8 +561,10 @@ export default class SearchHelper {
|
||||
|
||||
private static buildResultContext(document: Document, query: string) {
|
||||
// Reset regex lastIndex to avoid state issues with global regex
|
||||
this.QUOTED_QUERY_REGEX.lastIndex = 0;
|
||||
const quotedQueries = Array.from(query.matchAll(this.QUOTED_QUERY_REGEX));
|
||||
PostgresSearchProvider.QUOTED_QUERY_REGEX.lastIndex = 0;
|
||||
const quotedQueries = Array.from(
|
||||
query.matchAll(PostgresSearchProvider.QUOTED_QUERY_REGEX)
|
||||
);
|
||||
const text = DocumentHelper.toPlainText(document);
|
||||
|
||||
// Regex to highlight quoted queries as ts_headline will not do this by default due to stemming.
|
||||
@@ -562,7 +574,7 @@ export default class SearchHelper {
|
||||
fullMatchRegex.source,
|
||||
...(quotedQueries.length
|
||||
? quotedQueries.map((match) => escapeRegExp(match[1]))
|
||||
: this.removeStopWords(query)
|
||||
: PostgresSearchProvider.removeStopWords(query)
|
||||
.trim()
|
||||
.split(" ")
|
||||
.map((match) => `\\b${escapeRegExp(match)}\\b`)),
|
||||
@@ -571,8 +583,8 @@ export default class SearchHelper {
|
||||
);
|
||||
|
||||
// Reset regex lastIndex to avoid state issues with global regex
|
||||
this.BREAK_CHARS_REGEX.lastIndex = 0;
|
||||
const breakCharsRegex = this.BREAK_CHARS_REGEX;
|
||||
PostgresSearchProvider.BREAK_CHARS_REGEX.lastIndex = 0;
|
||||
const breakCharsRegex = PostgresSearchProvider.BREAK_CHARS_REGEX;
|
||||
|
||||
// chop text around the first match, prefer the first full match if possible.
|
||||
const fullMatchIndex = text.search(fullMatchRegex);
|
||||
@@ -715,15 +727,17 @@ export default class SearchHelper {
|
||||
let likelyUrls = getUrls(options.query);
|
||||
|
||||
// remove likely urls, and escape the rest of the query.
|
||||
let limitedQuery = this.escapeQuery(
|
||||
let limitedQuery = PostgresSearchProvider.escapeQuery(
|
||||
likelyUrls
|
||||
.reduce((q, url) => q.replace(url, ""), options.query)
|
||||
.slice(0, this.maxQueryLength)
|
||||
.slice(0, PostgresSearchProvider.maxQueryLength)
|
||||
.trim()
|
||||
);
|
||||
|
||||
// Escape the URLs
|
||||
likelyUrls = likelyUrls.map((url) => this.escapeQuery(url));
|
||||
likelyUrls = likelyUrls.map((url) =>
|
||||
PostgresSearchProvider.escapeQuery(url)
|
||||
);
|
||||
|
||||
// Extract quoted queries and add them to the where clause, up to a maximum of 3 total.
|
||||
const quotedQueries = Array.from(limitedQuery.matchAll(/"([^"]*)"/g)).map(
|
||||
@@ -785,7 +799,9 @@ export default class SearchHelper {
|
||||
|
||||
return {
|
||||
ranking: result.dataValues.searchRanking,
|
||||
context: query ? this.buildResultContext(document, query) : undefined,
|
||||
context: query
|
||||
? PostgresSearchProvider.buildResultContext(document, query)
|
||||
: undefined,
|
||||
document,
|
||||
};
|
||||
}),
|
||||
@@ -794,22 +810,26 @@ export default class SearchHelper {
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a user search query into a format that can be used by Postgres
|
||||
* Convert a user search query into a format that can be used by Postgres.
|
||||
*
|
||||
* @param query The user search query
|
||||
* @returns The query formatted for Postgres ts_query
|
||||
* @param query - the user search query.
|
||||
* @returns the query formatted for Postgres ts_query.
|
||||
*/
|
||||
public static webSearchQuery(query: string): string {
|
||||
// limit length of search queries as we're using regex against untrusted input
|
||||
let limitedQuery = this.escapeQuery(query.slice(0, this.maxQueryLength));
|
||||
let limitedQuery = PostgresSearchProvider.escapeQuery(
|
||||
query.slice(0, PostgresSearchProvider.maxQueryLength)
|
||||
);
|
||||
|
||||
const quotedSearch =
|
||||
limitedQuery.startsWith('"') && limitedQuery.endsWith('"');
|
||||
|
||||
// Replace single quote characters with &.
|
||||
// Reset regex lastIndex to avoid state issues with global regex
|
||||
this.SINGLE_QUOTE_REGEX.lastIndex = 0;
|
||||
const singleQuotes = limitedQuery.matchAll(this.SINGLE_QUOTE_REGEX);
|
||||
PostgresSearchProvider.SINGLE_QUOTE_REGEX.lastIndex = 0;
|
||||
const singleQuotes = limitedQuery.matchAll(
|
||||
PostgresSearchProvider.SINGLE_QUOTE_REGEX
|
||||
);
|
||||
|
||||
for (const match of singleQuotes) {
|
||||
if (
|
||||
@@ -851,11 +871,9 @@ export default class SearchHelper {
|
||||
}
|
||||
|
||||
private static removeStopWords(query: string): string {
|
||||
// Based on:
|
||||
// https://github.com/postgres/postgres/blob/fc0d0ce978752493868496be6558fa17b7c4c3cf/src/backend/snowball/stopwords/english.stop
|
||||
return query
|
||||
.split(" ")
|
||||
.filter((word) => !this.STOP_WORDS.has(word))
|
||||
.filter((word) => !PostgresSearchProvider.STOP_WORDS.has(word))
|
||||
.join(" ");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { PluginManager, Hook } from "@server/utils/PluginManager";
|
||||
import config from "../plugin.json";
|
||||
import PostgresSearchProvider from "./PostgresSearchProvider";
|
||||
|
||||
const provider = new PostgresSearchProvider();
|
||||
|
||||
PluginManager.add([
|
||||
{
|
||||
...config,
|
||||
type: Hook.SearchProvider,
|
||||
value: provider,
|
||||
},
|
||||
]);
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
AuthenticationProvider,
|
||||
Comment,
|
||||
} from "@server/models";
|
||||
import SearchHelper from "@server/models/helpers/SearchHelper";
|
||||
import SearchProviderManager from "@server/utils/SearchProviderManager";
|
||||
import { can } from "@server/policies";
|
||||
import type { APIContext } from "@server/types";
|
||||
import { safeEqual } from "@server/utils/crypto";
|
||||
@@ -238,7 +238,7 @@ router.post(
|
||||
return;
|
||||
}
|
||||
|
||||
const { results, total } = await SearchHelper.searchForUser(user, options);
|
||||
const { results, total } = await SearchProviderManager.getProvider().searchForUser(user, options);
|
||||
|
||||
await SearchQuery.create({
|
||||
userId: user ? user.id : null,
|
||||
|
||||
@@ -772,6 +772,14 @@ export class Environment {
|
||||
environment.ALLOWED_PRIVATE_IP_ADDRESSES
|
||||
);
|
||||
|
||||
/**
|
||||
* The search provider to use. Defaults to "postgres" which uses PostgreSQL
|
||||
* full-text search. Alternative providers can be registered via plugins.
|
||||
*/
|
||||
@IsOptional()
|
||||
public SEARCH_PROVIDER =
|
||||
this.toOptionalString(environment.SEARCH_PROVIDER) ?? "postgres";
|
||||
|
||||
/**
|
||||
* The product name
|
||||
*/
|
||||
|
||||
@@ -16,7 +16,7 @@ describe("DocumentArchivedProcessor", () => {
|
||||
userId: user.id,
|
||||
documentId: document.id,
|
||||
});
|
||||
|
||||
|
||||
// Verify the star exists
|
||||
expect(
|
||||
await Star.count({
|
||||
@@ -56,7 +56,7 @@ describe("DocumentArchivedProcessor", () => {
|
||||
teamId: actor.teamId,
|
||||
userId: actor.id,
|
||||
});
|
||||
|
||||
|
||||
// Create stars for both users
|
||||
await buildStar({
|
||||
userId: actor.id,
|
||||
@@ -95,7 +95,7 @@ describe("DocumentArchivedProcessor", () => {
|
||||
},
|
||||
})
|
||||
).toBe(0);
|
||||
|
||||
|
||||
// Verify the other user's star still exists
|
||||
expect(
|
||||
await Star.count({
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { SearchableModel } from "@shared/types";
|
||||
import {
|
||||
buildDocument,
|
||||
buildCollection,
|
||||
buildUser,
|
||||
} from "@server/test/factories";
|
||||
import SearchProviderManager from "@server/utils/SearchProviderManager";
|
||||
import SearchIndexProcessor from "./SearchIndexProcessor";
|
||||
|
||||
const processor = new SearchIndexProcessor();
|
||||
|
||||
describe("SearchIndexProcessor", () => {
|
||||
it("should have the expected applicable events", () => {
|
||||
expect(SearchIndexProcessor.applicableEvents).toContain(
|
||||
"documents.publish"
|
||||
);
|
||||
expect(SearchIndexProcessor.applicableEvents).toContain(
|
||||
"documents.update.delayed"
|
||||
);
|
||||
expect(SearchIndexProcessor.applicableEvents).toContain(
|
||||
"documents.permanent_delete"
|
||||
);
|
||||
expect(SearchIndexProcessor.applicableEvents).toContain(
|
||||
"collections.create"
|
||||
);
|
||||
expect(SearchIndexProcessor.applicableEvents).toContain("comments.create");
|
||||
expect(SearchIndexProcessor.applicableEvents).toContain("comments.delete");
|
||||
});
|
||||
|
||||
it("should call provider.index for documents.publish", async () => {
|
||||
const user = await buildUser();
|
||||
const collection = await buildCollection({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: user.teamId,
|
||||
collectionId: collection.id,
|
||||
userId: user.id,
|
||||
});
|
||||
|
||||
const provider = SearchProviderManager.getProvider();
|
||||
const indexSpy = jest.spyOn(provider, "index");
|
||||
|
||||
await processor.perform({
|
||||
name: "documents.publish",
|
||||
documentId: document.id,
|
||||
collectionId: collection.id,
|
||||
teamId: user.teamId,
|
||||
actorId: user.id,
|
||||
} as any);
|
||||
|
||||
expect(indexSpy).toHaveBeenCalledWith(
|
||||
SearchableModel.Document,
|
||||
expect.objectContaining({ id: document.id })
|
||||
);
|
||||
|
||||
indexSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("should call provider.remove for documents.permanent_delete", async () => {
|
||||
const user = await buildUser();
|
||||
const provider = SearchProviderManager.getProvider();
|
||||
const removeSpy = jest.spyOn(provider, "remove");
|
||||
|
||||
await processor.perform({
|
||||
name: "documents.permanent_delete",
|
||||
documentId: "deleted-doc-id",
|
||||
collectionId: "some-collection-id",
|
||||
teamId: user.teamId,
|
||||
actorId: user.id,
|
||||
} as any);
|
||||
|
||||
expect(removeSpy).toHaveBeenCalledWith(
|
||||
SearchableModel.Document,
|
||||
"deleted-doc-id",
|
||||
user.teamId
|
||||
);
|
||||
|
||||
removeSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,139 @@
|
||||
import { SearchableModel } from "@shared/types";
|
||||
import { Document, Collection, Comment } from "@server/models";
|
||||
import BaseProcessor from "@server/queues/processors/BaseProcessor";
|
||||
import type {
|
||||
DocumentEvent,
|
||||
DocumentMovedEvent,
|
||||
CollectionEvent,
|
||||
CommentEvent,
|
||||
CommentUpdateEvent,
|
||||
Event,
|
||||
} from "@server/types";
|
||||
import SearchProviderManager from "@server/utils/SearchProviderManager";
|
||||
|
||||
/**
|
||||
* Processor that keeps the search index in sync with data changes.
|
||||
* For PostgreSQL this is largely a no-op since tsvector triggers handle
|
||||
* indexing, but external providers (Elasticsearch, etc.) rely on these
|
||||
* events to maintain their indexes.
|
||||
*/
|
||||
export default class SearchIndexProcessor extends BaseProcessor {
|
||||
static applicableEvents: Event["name"][] = [
|
||||
"documents.publish",
|
||||
"documents.update.delayed",
|
||||
"documents.archive",
|
||||
"documents.unarchive",
|
||||
"documents.delete",
|
||||
"documents.permanent_delete",
|
||||
"documents.move",
|
||||
"collections.create",
|
||||
"collections.update",
|
||||
"collections.delete",
|
||||
"comments.create",
|
||||
"comments.update",
|
||||
"comments.delete",
|
||||
];
|
||||
|
||||
async perform(
|
||||
event: DocumentEvent | DocumentMovedEvent | CollectionEvent | CommentEvent
|
||||
): Promise<void> {
|
||||
const provider = SearchProviderManager.getProvider();
|
||||
|
||||
// When using the built-in Postgres search provider, tsvector triggers
|
||||
// handle indexing directly and the provider methods are effectively no-ops for now.
|
||||
if (process.env.SEARCH_PROVIDER === "postgres") {
|
||||
return;
|
||||
}
|
||||
|
||||
switch (event.name) {
|
||||
case "documents.publish":
|
||||
case "documents.update.delayed":
|
||||
case "documents.unarchive": {
|
||||
const document = await Document.findByPk(
|
||||
(event as DocumentEvent).documentId
|
||||
);
|
||||
if (document) {
|
||||
await provider.index(SearchableModel.Document, document);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "documents.archive":
|
||||
case "documents.delete": {
|
||||
const document = await Document.findByPk(
|
||||
(event as DocumentEvent).documentId,
|
||||
{ paranoid: false }
|
||||
);
|
||||
if (document) {
|
||||
await provider.updateMetadata(SearchableModel.Document, document.id, {
|
||||
archivedAt: document.archivedAt,
|
||||
deletedAt: document.deletedAt,
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "documents.permanent_delete": {
|
||||
await provider.remove(
|
||||
SearchableModel.Document,
|
||||
(event as DocumentEvent).documentId,
|
||||
event.teamId
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
case "documents.move": {
|
||||
const movedEvent = event as DocumentMovedEvent;
|
||||
for (const documentId of movedEvent.data.documentIds) {
|
||||
await provider.updateMetadata(SearchableModel.Document, documentId, {
|
||||
collectionId: movedEvent.collectionId,
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "collections.create":
|
||||
case "collections.update": {
|
||||
const collection = await Collection.findByPk(
|
||||
(event as CollectionEvent).collectionId
|
||||
);
|
||||
if (collection) {
|
||||
await provider.index(SearchableModel.Collection, collection);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "collections.delete": {
|
||||
await provider.remove(
|
||||
SearchableModel.Collection,
|
||||
(event as CollectionEvent).collectionId,
|
||||
event.teamId
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
case "comments.create":
|
||||
case "comments.update": {
|
||||
const comment = await Comment.findByPk(
|
||||
(event as CommentEvent | CommentUpdateEvent).modelId
|
||||
);
|
||||
if (comment) {
|
||||
await provider.index(SearchableModel.Comment, comment);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "comments.delete": {
|
||||
await provider.remove(
|
||||
SearchableModel.Comment,
|
||||
(event as CommentEvent).modelId,
|
||||
event.teamId
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,7 +64,7 @@ import {
|
||||
import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
|
||||
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
|
||||
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
|
||||
import SearchHelper from "@server/models/helpers/SearchHelper";
|
||||
import SearchProviderManager from "@server/utils/SearchProviderManager";
|
||||
import { TextHelper } from "@server/models/helpers/TextHelper";
|
||||
import { authorize, cannot } from "@server/policies";
|
||||
import {
|
||||
@@ -1015,17 +1015,18 @@ router.post(
|
||||
collaboratorIds = [userId];
|
||||
}
|
||||
|
||||
const documents = await SearchHelper.searchTitlesForUser(user, {
|
||||
query,
|
||||
dateFilter,
|
||||
statusFilter,
|
||||
collectionId,
|
||||
collaboratorIds,
|
||||
offset,
|
||||
limit,
|
||||
sort: sort as SortFilter,
|
||||
direction: direction as DirectionFilter,
|
||||
});
|
||||
const documents =
|
||||
await SearchProviderManager.getProvider().searchTitlesForUser(user, {
|
||||
query,
|
||||
dateFilter,
|
||||
statusFilter,
|
||||
collectionId,
|
||||
collaboratorIds,
|
||||
offset,
|
||||
limit,
|
||||
sort: sort as SortFilter,
|
||||
direction: direction as DirectionFilter,
|
||||
});
|
||||
const policies = presentPolicies(user, documents);
|
||||
const data = await presentDocuments(ctx, documents);
|
||||
|
||||
@@ -1099,7 +1100,7 @@ router.post(
|
||||
const team = await share.$get("team");
|
||||
invariant(team, "Share must belong to a team");
|
||||
|
||||
response = await SearchHelper.searchForTeam(team, {
|
||||
response = await SearchProviderManager.getProvider().searchForTeam(team, {
|
||||
query,
|
||||
collectionId: collection?.id || document?.collectionId,
|
||||
share,
|
||||
@@ -1145,7 +1146,7 @@ router.post(
|
||||
collaboratorIds = [userId];
|
||||
}
|
||||
|
||||
response = await SearchHelper.searchForUser(user, {
|
||||
response = await SearchProviderManager.getProvider().searchForUser(user, {
|
||||
query,
|
||||
collaboratorIds,
|
||||
collectionId,
|
||||
|
||||
@@ -5,7 +5,7 @@ import { StatusFilter } from "@shared/types";
|
||||
import auth from "@server/middlewares/authentication";
|
||||
import validate from "@server/middlewares/validate";
|
||||
import { Group, User } from "@server/models";
|
||||
import SearchHelper from "@server/models/helpers/SearchHelper";
|
||||
import SearchProviderManager from "@server/utils/SearchProviderManager";
|
||||
import { can } from "@server/policies";
|
||||
import {
|
||||
presentDocuments,
|
||||
@@ -29,7 +29,7 @@ router.post(
|
||||
const actor = ctx.state.auth.user;
|
||||
|
||||
const [documents, users, groups, collections] = await Promise.all([
|
||||
SearchHelper.searchTitlesForUser(actor, {
|
||||
SearchProviderManager.getProvider().searchTitlesForUser(actor, {
|
||||
query,
|
||||
offset,
|
||||
limit,
|
||||
@@ -74,7 +74,7 @@ router.post(
|
||||
offset,
|
||||
limit,
|
||||
}),
|
||||
SearchHelper.searchCollectionsForUser(actor, { query, offset, limit }),
|
||||
SearchProviderManager.getProvider().searchCollectionsForUser(actor, { query, offset, limit }),
|
||||
]);
|
||||
|
||||
ctx.body = {
|
||||
|
||||
@@ -7,7 +7,6 @@ import documentUpdater from "@server/commands/documentUpdater";
|
||||
import { Op } from "sequelize";
|
||||
import { Collection, Document } from "@server/models";
|
||||
import { sequelize } from "@server/storage/database";
|
||||
import SearchHelper from "@server/models/helpers/SearchHelper";
|
||||
import { authorize } from "@server/policies";
|
||||
import { presentDocument } from "@server/presenters";
|
||||
import AuthenticationHelper from "@shared/helpers/AuthenticationHelper";
|
||||
@@ -22,6 +21,7 @@ import {
|
||||
withTracing,
|
||||
} from "./util";
|
||||
import { TextEditMode } from "@shared/types";
|
||||
import SearchProviderManager from "@server/utils/SearchProviderManager";
|
||||
|
||||
/**
|
||||
* Registers document-related MCP tools on the given server, filtered by
|
||||
@@ -93,6 +93,8 @@ export function documentTools(server: McpServer, scopes: string[]) {
|
||||
}
|
||||
|
||||
if (query) {
|
||||
const searchProvider = SearchProviderManager.getProvider();
|
||||
|
||||
// If the query looks like a document ID or urlId, try direct
|
||||
// lookup first so exact matches appear at the top of results.
|
||||
let exactMatch: Document | null = null;
|
||||
@@ -109,7 +111,7 @@ export function documentTools(server: McpServer, scopes: string[]) {
|
||||
}
|
||||
}
|
||||
|
||||
const { results } = await SearchHelper.searchForUser(user, {
|
||||
const { results } = await searchProvider.searchForUser(user, {
|
||||
query,
|
||||
collectionId,
|
||||
offset: effectiveOffset,
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { DateFilter } from "@shared/types";
|
||||
import type { SearchableModel } from "@shared/types";
|
||||
import type { DirectionFilter, SortFilter, StatusFilter } from "@shared/types";
|
||||
import type Collection from "@server/models/Collection";
|
||||
import type Comment from "@server/models/Comment";
|
||||
import type Document from "@server/models/Document";
|
||||
import type Share from "@server/models/Share";
|
||||
import type Team from "@server/models/Team";
|
||||
import type User from "@server/models/User";
|
||||
|
||||
export interface SearchResponse {
|
||||
results: {
|
||||
/** The search ranking, for sorting results. */
|
||||
ranking: number;
|
||||
/** A snippet of contextual text around the search result. */
|
||||
context?: string;
|
||||
/** The document result. */
|
||||
document: Document;
|
||||
}[];
|
||||
/** The total number of results for the search query without pagination. */
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SearchOptions {
|
||||
/** The query limit for pagination. */
|
||||
limit?: number;
|
||||
/** The query offset for pagination. */
|
||||
offset?: number;
|
||||
/** The text to search for. */
|
||||
query?: string;
|
||||
/** Limit results to a collection. Authorization is presumed to have been done before passing to this provider. */
|
||||
collectionId?: string | null;
|
||||
/** Limit results to a shared document. */
|
||||
share?: Share;
|
||||
/** Limit results to a date range. */
|
||||
dateFilter?: DateFilter;
|
||||
/** Status of the documents to return. */
|
||||
statusFilter?: StatusFilter[];
|
||||
/** Limit results to a list of documents. */
|
||||
documentIds?: string[];
|
||||
/** Limit results to a list of users that collaborated on the document. */
|
||||
collaboratorIds?: string[];
|
||||
/** The minimum number of words to be returned in the contextual snippet. */
|
||||
snippetMinWords?: number;
|
||||
/** The maximum number of words to be returned in the contextual snippet. */
|
||||
snippetMaxWords?: number;
|
||||
/** The field to sort results by. */
|
||||
sort?: SortFilter;
|
||||
/** The sort direction. */
|
||||
direction?: DirectionFilter;
|
||||
/** Whether to boost results by popularity score. Defaults to true. */
|
||||
usePopularityBoost?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract base class for search providers. Implementations handle full-text
|
||||
* search, title search, collection search, and index management.
|
||||
*/
|
||||
export abstract class BaseSearchProvider {
|
||||
/** Unique identifier for this provider, matched against `SEARCH_PROVIDER` env var. */
|
||||
abstract id: string;
|
||||
|
||||
/**
|
||||
* Perform a full-text search scoped to a user's accessible documents.
|
||||
*
|
||||
* @param user - the user performing the search.
|
||||
* @param options - search options.
|
||||
* @returns search results with ranking and context.
|
||||
*/
|
||||
abstract searchForUser(
|
||||
user: User,
|
||||
options?: SearchOptions
|
||||
): Promise<SearchResponse>;
|
||||
|
||||
/**
|
||||
* Perform a full-text search scoped to a team (used for shared document search).
|
||||
*
|
||||
* @param team - the team to search within.
|
||||
* @param options - search options.
|
||||
* @returns search results with ranking and context.
|
||||
*/
|
||||
abstract searchForTeam(
|
||||
team: Team,
|
||||
options?: SearchOptions
|
||||
): Promise<SearchResponse>;
|
||||
|
||||
/**
|
||||
* Search document titles for a user (used for link suggestions, quick search).
|
||||
*
|
||||
* @param user - the user performing the search.
|
||||
* @param options - search options.
|
||||
* @returns matching documents.
|
||||
*/
|
||||
abstract searchTitlesForUser(
|
||||
user: User,
|
||||
options?: SearchOptions
|
||||
): Promise<Document[]>;
|
||||
|
||||
/**
|
||||
* Search collections for a user.
|
||||
*
|
||||
* @param user - the user performing the search.
|
||||
* @param options - search options.
|
||||
* @returns matching collections.
|
||||
*/
|
||||
abstract searchCollectionsForUser(
|
||||
user: User,
|
||||
options?: SearchOptions
|
||||
): Promise<Collection[]>;
|
||||
|
||||
/**
|
||||
* Index or re-index a searchable item. For providers that rely on database
|
||||
* triggers (e.g. PostgreSQL tsvector), this may be a no-op.
|
||||
*
|
||||
* @param model - the type of model being indexed.
|
||||
* @param item - the model instance to index.
|
||||
*/
|
||||
abstract index(
|
||||
model: SearchableModel,
|
||||
item: Document | Collection | Comment
|
||||
): Promise<void>;
|
||||
|
||||
/**
|
||||
* Remove an item from the search index.
|
||||
*
|
||||
* @param model - the type of model being removed.
|
||||
* @param id - the id of the item to remove.
|
||||
* @param teamId - the team id the item belongs to.
|
||||
*/
|
||||
abstract remove(
|
||||
model: SearchableModel,
|
||||
id: string,
|
||||
teamId: string
|
||||
): Promise<void>;
|
||||
|
||||
/**
|
||||
* Update metadata for an indexed item without re-indexing the full content.
|
||||
* Useful for permission changes, moves, archive/unarchive.
|
||||
*
|
||||
* @param model - the type of model being updated.
|
||||
* @param id - the id of the item to update.
|
||||
* @param metadata - the metadata fields to update.
|
||||
*/
|
||||
abstract updateMetadata(
|
||||
model: SearchableModel,
|
||||
id: string,
|
||||
metadata: Record<string, unknown>
|
||||
): Promise<void>;
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import type { BaseTask } from "@server/queues/tasks/base/BaseTask";
|
||||
import type { UnfurlSignature, UninstallSignature } from "@server/types";
|
||||
import type { BaseIssueProvider } from "./BaseIssueProvider";
|
||||
import type { GroupSyncProvider } from "./GroupSyncProvider";
|
||||
import type { BaseSearchProvider } from "./BaseSearchProvider";
|
||||
|
||||
export enum PluginPriority {
|
||||
VeryHigh = 0,
|
||||
@@ -29,6 +30,7 @@ export enum Hook {
|
||||
EmailTemplate = "emailTemplate",
|
||||
IssueProvider = "issueProvider",
|
||||
Processor = "processor",
|
||||
SearchProvider = "searchProvider",
|
||||
Task = "task",
|
||||
UnfurlProvider = "unfurl",
|
||||
Uninstall = "uninstall",
|
||||
@@ -45,6 +47,7 @@ type PluginValueMap = {
|
||||
[Hook.EmailTemplate]: typeof BaseEmail<any>;
|
||||
[Hook.IssueProvider]: BaseIssueProvider;
|
||||
[Hook.Processor]: typeof BaseProcessor;
|
||||
[Hook.SearchProvider]: BaseSearchProvider;
|
||||
[Hook.Task]: typeof BaseTask<any>;
|
||||
[Hook.Uninstall]: UninstallSignature;
|
||||
[Hook.UnfurlProvider]: { unfurl: UnfurlSignature; cacheExpiry: number };
|
||||
@@ -106,9 +109,10 @@ export class PluginManager {
|
||||
|
||||
/**
|
||||
* Returns all the plugins of a given type in order of priority.
|
||||
* Triggers loading of all plugins from disk if not already loaded.
|
||||
*
|
||||
* @param type The type of plugin to filter by
|
||||
* @returns A list of plugins
|
||||
* @param type - the type of plugin to filter by.
|
||||
* @returns a list of plugins.
|
||||
*/
|
||||
public static getHooks<T extends Hook>(type: T) {
|
||||
this.loadPlugins();
|
||||
@@ -139,9 +143,9 @@ export class PluginManager {
|
||||
|
||||
glob
|
||||
.sync(path.join(rootDir, "plugins/*/server/!(*.test|schema).[jt]s"))
|
||||
.forEach((filePath: string) => {
|
||||
require(path.join(process.cwd(), filePath));
|
||||
});
|
||||
.forEach((filePath: string) =>
|
||||
require(path.join(process.cwd(), filePath))
|
||||
);
|
||||
this.loaded = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import env from "@server/env";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import type { BaseSearchProvider } from "./BaseSearchProvider";
|
||||
import { Hook, PluginManager } from "./PluginManager";
|
||||
|
||||
/**
|
||||
* Manages selection and caching of the active search provider based on the
|
||||
* `SEARCH_PROVIDER` environment variable.
|
||||
*/
|
||||
export default class SearchProviderManager {
|
||||
private static cachedProvider: BaseSearchProvider | undefined;
|
||||
|
||||
/**
|
||||
* Returns the active search provider. The provider is determined by matching
|
||||
* `SEARCH_PROVIDER` env var against registered `Hook.SearchProvider` plugins.
|
||||
*
|
||||
* @returns the active search provider instance.
|
||||
* @throws if no matching provider is found.
|
||||
*/
|
||||
public static getProvider(): BaseSearchProvider {
|
||||
if (this.cachedProvider) {
|
||||
return this.cachedProvider;
|
||||
}
|
||||
|
||||
const providerId = env.SEARCH_PROVIDER;
|
||||
const plugins = PluginManager.getHooks(Hook.SearchProvider);
|
||||
|
||||
for (const plugin of plugins) {
|
||||
if (plugin.value.id === providerId) {
|
||||
this.cachedProvider = plugin.value;
|
||||
Logger.debug("plugins", `Using search provider: ${plugin.value.id}`);
|
||||
return this.cachedProvider;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Search provider "${providerId}" not found. Available providers: ${plugins
|
||||
.map((p) => p.value.id)
|
||||
.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the cached provider. Useful for testing.
|
||||
*/
|
||||
public static reset(): void {
|
||||
this.cachedProvider = undefined;
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,13 @@ export enum DirectionFilter {
|
||||
DESC = "DESC",
|
||||
}
|
||||
|
||||
/** Model types that support search indexing. */
|
||||
export enum SearchableModel {
|
||||
Document = "document",
|
||||
Collection = "collection",
|
||||
Comment = "comment",
|
||||
}
|
||||
|
||||
export enum CollectionStatusFilter {
|
||||
Archived = "archived",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user