feat: Add sitemap to publicly shared documents with indexing enabled (#9334)

* quick: Add sitemap to publicly shared documents with indexing enabled

* escape

app/scenes/Document/Shared.tsx

@@ -195,6 +195,11 @@ function SharedDocumentScene(props: Props) {
           rel="canonical"
           href={canonicalOrigin + location.pathname.replace(/\/$/, "")}
         />
+        <link
+          rel="sitemap"
+          type="application/xml"
+          href={`${env.URL}/api/documents.sitemap?shareId=${shareId}`}
+        />
       </Helmet>
       <TeamContext.Provider value={response.team}>
         <ThemeProvider theme={theme}>

server/routes/api/documents/documents.ts

@@ -73,6 +73,7 @@ import { APIContext } from "@server/types";
 import { RateLimiterStrategy } from "@server/utils/RateLimiter";
 import ZipHelper from "@server/utils/ZipHelper";
 import { getTeamFromContext } from "@server/utils/passport";
+import { navigationNodeToSitemap } from "@server/utils/sitemap";
 import { assertPresent } from "@server/validation";
 import pagination from "../middlewares/pagination";
 import * as T from "./schema";
@@ -691,6 +692,29 @@ router.post(
   }
 );
 
+router.get(
+  "documents.sitemap",
+  rateLimiter(RateLimiterStrategy.TwentyFivePerMinute),
+  auth({ optional: true }),
+  validate(T.DocumentsSitemapSchema),
+  async (ctx: APIContext<T.DocumentsSitemapReq>) => {
+    const { shareId } = ctx.input.query;
+    const { collection, share } = await documentLoader({
+      shareId,
+    });
+
+    let tree;
+    if (share && share.includeChildDocuments && share.allowIndexing) {
+      tree = collection?.getDocumentTree(share.documentId);
+    }
+
+    const baseUrl = `${process.env.URL}/s/${shareId}`;
+
+    ctx.set("Content-Type", "application/xml");
+    ctx.body = navigationNodeToSitemap(tree, baseUrl);
+  }
+);
+
 router.post(
   "documents.export",
   rateLimiter(RateLimiterStrategy.TwentyFivePerMinute),

server/routes/api/documents/schema.ts

@@ -452,3 +452,11 @@ export const DocumentsMembershipsSchema = BaseSchema.extend({
 export type DocumentsMembershipsReq = z.infer<
   typeof DocumentsMembershipsSchema
 >;
+
+export const DocumentsSitemapSchema = BaseSchema.extend({
+  query: z.object({
+    shareId: z.string(),
+  }),
+});
+
+export type DocumentsSitemapReq = z.infer<typeof DocumentsSitemapSchema>;

server/utils/sitemap.ts

@@ -0,0 +1,35 @@
+import escape from "lodash/escape";
+import { NavigationNode } from "@shared/types";
+
+/**
+ * Converts a navigation tree to a sitemap XML string, by traversing the nodes.
+ *
+ * @param tree The navigation tree to convert.
+ * @param baseUrl The base URL to prepend to each node's URL.
+ * @returns The sitemap XML string.
+ */
+export function navigationNodeToSitemap(
+  tree: NavigationNode | undefined | null,
+  baseUrl: string
+): string {
+  const urls: string[] = [];
+
+  function collectUrls(node: NavigationNode, urls: string[]) {
+    urls.push(`${baseUrl}${node.url}`);
+    if (node.children) {
+      node.children.forEach((child) => collectUrls(child, urls));
+    }
+  }
+
+  if (tree) {
+    collectUrls(tree, urls);
+  }
+
+  // Build XML
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls
+    .map(
+      (url) =>
+        `  <url><loc>${escape(url)}</loc><changefreq>weekly</changefreq></url>`
+    )
+    .join("\n")}\n</urlset>`;
+}