Listen to GitHub webhooks to update issueSources cache (#9414)

* Listen to GitHub webhooks to update issue-sources cache

* Add `GitHubWebhookTask`

* review

.env.sample

@@ -202,6 +202,7 @@ RATE_LIMITER_DURATION_WINDOW=60
 # DOCS: https://docs.getoutline.com/s/hosting/doc/github-GchT3NNxI9
 GITHUB_CLIENT_ID=
 GITHUB_CLIENT_SECRET=
+GITHUB_WEBHOOK_SECRET=
 GITHUB_APP_NAME=
 GITHUB_APP_ID=
 GITHUB_APP_PRIVATE_KEY=

package.json

@@ -86,6 +86,7 @@
     "@node-oauth/oauth2-server": "^5.2.0",
     "@notionhq/client": "^2.3.0",
     "@octokit/auth-app": "^6.1.4",
+    "@octokit/webhooks": "^13.8.0",
     "@outlinewiki/koa-passport": "^4.2.1",
     "@outlinewiki/passport-azure-ad-oauth2": "^0.1.0",
     "@radix-ui/react-collapsible": "^1.1.11",

plugins/github/server/GitHubIssueProvider.ts

@@ -1,7 +1,14 @@
 import { Endpoints } from "@octokit/types";
+import {
+  InstallationNewPermissionsAcceptedEvent,
+  InstallationRepositoriesEvent,
+  RepositoryRenamedEvent,
+} from "@octokit/webhooks-types";
 import { IssueSource } from "@shared/schema";
 import { IntegrationService, IntegrationType } from "@shared/types";
-import { Integration } from "@server/models";
+import Logger from "@server/logging/Logger";
+import { Integration, IntegrationAuthentication } from "@server/models";
+import { sequelize } from "@server/storage/database";
 import { BaseIssueProvider } from "@server/utils/BaseIssueProvider";
 import { GitHub } from "./github";
 
@@ -37,4 +44,215 @@ export class GitHubIssueProvider extends BaseIssueProvider {
 
     return sources;
   }
+
+  async handleWebhook({
+    payload,
+    headers,
+  }: {
+    payload: Record<string, unknown>;
+    headers: Record<string, unknown>;
+  }) {
+    const hookId = headers["x-github-hook-id"] as string;
+    const eventName = headers["x-github-event"] as string;
+    const action = payload.action as string;
+
+    if (!eventName || !action) {
+      Logger.warn(
+        `Received GitHub webhook without event name or action; hookId: ${hookId}, eventName: ${eventName}, action: ${action}`
+      );
+      return;
+    }
+
+    switch (eventName) {
+      case "installation": {
+        await this.handleInstallationEvent(payload, action);
+        break;
+      }
+
+      case "installation_repositories": {
+        await this.handleInstallationRepositoriesEvent(
+          payload as unknown as InstallationRepositoriesEvent
+        );
+        break;
+      }
+
+      case "repository": {
+        await this.handleRepositoryEvent(payload, action, hookId);
+        break;
+      }
+
+      default:
+    }
+  }
+
+  private async handleInstallationEvent(
+    payload: Record<string, unknown>,
+    action: string
+  ): Promise<void> {
+    if (action !== "new_permissions_accepted") {
+      return;
+    }
+
+    const event = payload as unknown as InstallationNewPermissionsAcceptedEvent;
+    const installationId = event.installation.id;
+    const integration = await Integration.findOne({
+      where: {
+        service: IntegrationService.GitHub,
+        "settings.github.installation.id": installationId,
+      },
+    });
+
+    if (!integration) {
+      Logger.warn(
+        `GitHub installation new_permissions_accepted event without integration; installationId: ${installationId}`
+      );
+      return;
+    }
+
+    const sources = await this.fetchSources(integration);
+
+    const client = await GitHub.authenticateAsInstallation(installationId);
+    const installation = await client.requestAppInstallation(installationId);
+
+    const scopes = Object.entries(installation.data.permissions).map(
+      ([name, permission]) => `${name}:${permission}`
+    );
+
+    await sequelize.transaction(async (transaction) => {
+      await integration.reload({
+        include: {
+          model: IntegrationAuthentication,
+          as: "authentication",
+          required: true,
+        },
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+      const authentication = integration.authentication;
+
+      if (!authentication) {
+        Logger.warn(
+          `GitHub integration without authentication; integrationId: ${integration.id}`
+        );
+        return;
+      }
+
+      authentication.scopes = scopes;
+      await authentication.save({ transaction });
+
+      integration.issueSources = sources;
+      integration.changed("issueSources", true);
+      await integration.save({ transaction });
+    });
+  }
+
+  private async handleInstallationRepositoriesEvent(
+    event: InstallationRepositoriesEvent
+  ): Promise<void> {
+    const installationId = event.installation.id;
+    const account = event.installation.account;
+
+    await sequelize.transaction(async (transaction) => {
+      const integration = await Integration.findOne({
+        where: {
+          service: IntegrationService.GitHub,
+          "settings.github.installation.id": installationId,
+        },
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+
+      if (!integration) {
+        Logger.warn(
+          `GitHub installation_repositories event without integration; installationId: ${installationId}`
+        );
+        return;
+      }
+
+      let sources = integration.issueSources ?? [];
+
+      if (event.action === "added") {
+        const addedSources = event.repositories_added.map<IssueSource>(
+          (repo) => ({
+            id: String(repo.id),
+            name: repo.name,
+            owner: {
+              id: String(account.id),
+              name: account.login,
+            },
+            service: IntegrationService.GitHub,
+          })
+        );
+        sources.push(...addedSources);
+      } else {
+        const removedSourceIds = event.repositories_removed.map((repo) =>
+          String(repo.id)
+        );
+        sources = sources.filter(
+          (source) => !removedSourceIds.includes(source.id)
+        );
+      }
+
+      integration.issueSources = sources;
+      integration.changed("issueSources", true);
+      await integration.save({ transaction });
+    });
+  }
+
+  private async handleRepositoryEvent(
+    payload: Record<string, unknown>,
+    action: string,
+    hookId: string
+  ): Promise<void> {
+    if (action !== "renamed") {
+      return;
+    }
+
+    const event = payload as unknown as RepositoryRenamedEvent;
+    const installationId = event.installation?.id;
+
+    if (!installationId) {
+      Logger.warn(
+        `GitHub repository renamed event without installation ID; hookId: ${hookId}`
+      );
+      return;
+    }
+
+    const repoId = event.repository.id;
+    const repoName = event.repository.name;
+
+    await sequelize.transaction(async (transaction) => {
+      const integration = await Integration.findOne({
+        where: {
+          service: IntegrationService.GitHub,
+          "settings.github.installation.id": installationId,
+        },
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+
+      if (!integration) {
+        Logger.warn(
+          `GitHub repository renamed event without integration; installationId: ${installationId}`
+        );
+        return;
+      }
+
+      const source = integration.issueSources?.find(
+        (s) => s.id === String(repoId)
+      );
+
+      if (!source) {
+        Logger.info(
+          "task",
+          `No matching issue source found for repository ID: ${repoId}, integration ID: ${integration.id}`
+        );
+        return;
+      }
+
+      source.name = repoName;
+      integration.changed("issueSources", true);
+      await integration.save({ transaction });
+    });
+  }
 }

plugins/github/server/api/github.ts

@@ -6,10 +6,13 @@ import apexAuthRedirect from "@server/middlewares/apexAuthRedirect";
 import auth from "@server/middlewares/authentication";
 import { transaction } from "@server/middlewares/transaction";
 import validate from "@server/middlewares/validate";
+import validateWebhook from "@server/middlewares/validateWebhook";
 import { IntegrationAuthentication, Integration } from "@server/models";
 import { APIContext } from "@server/types";
 import { GitHubUtils } from "../../shared/GitHubUtils";
+import env from "../env";
 import { GitHub } from "../github";
+import GitHubWebhookTask from "../tasks/GitHubWebhookTask";
 import * as T from "./schema";
 
 const router = new Router();
@@ -60,11 +63,16 @@ router.get(
       return ctx.redirect(GitHubUtils.errorUrl("unauthenticated"));
     }
 
+    const scopes = Object.entries(installation.permissions).map(
+      ([name, permission]) => `${name}:${permission}`
+    );
+
     const authentication = await IntegrationAuthentication.create(
       {
         service: IntegrationService.GitHub,
         userId: user.id,
         teamId: user.teamId,
+        scopes,
       },
       { transaction }
     );
@@ -92,4 +100,29 @@ router.get(
   }
 );
 
+router.post(
+  "github.webhooks",
+  validateWebhook({
+    secretKey: env.GITHUB_WEBHOOK_SECRET!,
+    getSignatureFromHeader: (ctx) => {
+      const { headers } = ctx.request;
+      const signatureHeader = headers["x-hub-signature-256"];
+      const signature = Array.isArray(signatureHeader)
+        ? signatureHeader[0]
+        : signatureHeader;
+      return signature?.split("=")[1];
+    },
+  }),
+  async (ctx: APIContext) => {
+    const { headers, body } = ctx.request;
+
+    await new GitHubWebhookTask().schedule({
+      payload: body,
+      headers,
+    });
+
+    ctx.status = 202;
+  }
+);
+
 export default router;

plugins/github/server/env.ts

@@ -26,6 +26,12 @@ class GitHubPluginEnvironment extends Environment {
     environment.GITHUB_CLIENT_SECRET
   );
 
+  @IsOptional()
+  @CannotUseWithout("GITHUB_CLIENT_ID")
+  public GITHUB_WEBHOOK_SECRET = this.toOptionalString(
+    environment.GITHUB_WEBHOOK_SECRET
+  );
+
   @IsOptional()
   @CannotUseWithout("GITHUB_APP_PRIVATE_KEY")
   public GITHUB_APP_ID = this.toOptionalString(environment.GITHUB_APP_ID);

plugins/github/server/github.ts

@@ -22,6 +22,8 @@ type PR =
   Endpoints["GET /repos/{owner}/{repo}/pulls/{pull_number}"]["response"]["data"];
 type Issue =
   Endpoints["GET /repos/{owner}/{repo}/issues/{issue_number}"]["response"]["data"];
+type Installation =
+  Endpoints["GET /app/installations/{installation_id}"]["response"]["data"];
 
 const requestPlugin = (octokit: Octokit) => ({
   requestRepos: () =>
@@ -86,6 +88,23 @@ const requestPlugin = (octokit: Octokit) => ({
     }
   },
 
+  /**
+   * Fetches details of a specific GitHub app installation
+   *
+   * @param installationId Id of the installation to fetch
+   * @returns Response containing installation details
+   */
+  requestAppInstallation: async (
+    installationId: number
+  ): Promise<OctokitResponse<Installation>> =>
+    octokit.request("GET /app/installations/{installation_id}", {
+      installation_id: installationId,
+      headers: {
+        Accept: "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+      },
+    }),
+
   /**
    * Uninstalls the GitHub app from a given target
    *

plugins/github/server/index.ts

@@ -5,6 +5,7 @@ import { GitHubIssueProvider } from "./GitHubIssueProvider";
 import router from "./api/github";
 import env from "./env";
 import { GitHub } from "./github";
+import GitHubWebhookTask from "./tasks/GitHubWebhookTask";
 import { uninstall } from "./uninstall";
 
 const enabled =
@@ -21,6 +22,10 @@ if (enabled) {
       type: Hook.API,
       value: router,
     },
+    {
+      type: Hook.Task,
+      value: GitHubWebhookTask,
+    },
     {
       type: Hook.IssueProvider,
       value: new GitHubIssueProvider(),

plugins/github/server/tasks/GitHubWebhookTask.ts

@@ -0,0 +1,26 @@
+import { IntegrationService } from "@shared/types";
+import BaseTask from "@server/queues/tasks/BaseTask";
+import { Hook, PluginManager } from "@server/utils/PluginManager";
+
+type Props = {
+  headers: Record<string, unknown>;
+  payload: Record<string, unknown>;
+};
+
+export default class GitHubWebhookTask extends BaseTask<Props> {
+  public async perform({ headers, payload }: Props): Promise<void> {
+    const plugins = PluginManager.getHooks(Hook.IssueProvider);
+    const plugin = plugins.find(
+      (p) => p.value.service === IntegrationService.GitHub
+    );
+
+    if (!plugin) {
+      return;
+    }
+
+    await plugin.value.handleWebhook({
+      headers,
+      payload,
+    });
+  }
+}

server/middlewares/validateWebhook.ts

@@ -0,0 +1,36 @@
+import crypto from "crypto";
+import { Next } from "koa";
+import { APIContext } from "@server/types";
+import { safeEqual } from "@server/utils/crypto";
+
+export default function validateWebhook({
+  secretKey,
+  getSignatureFromHeader,
+}: {
+  secretKey: string;
+  getSignatureFromHeader: (ctx: APIContext) => string | undefined;
+}) {
+  return async function validateWebhookMiddleware(ctx: APIContext, next: Next) {
+    const { body } = ctx.request;
+    const signatureFromHeader = getSignatureFromHeader(ctx);
+
+    if (!signatureFromHeader) {
+      ctx.status = 401;
+      ctx.body = "Missing signature header";
+      return;
+    }
+
+    const computedSignature = crypto
+      .createHmac("sha256", secretKey)
+      .update(JSON.stringify(body))
+      .digest("hex");
+
+    if (!safeEqual(computedSignature, signatureFromHeader)) {
+      ctx.status = 401;
+      ctx.body = "Invalid signature";
+      return;
+    }
+
+    return next();
+  };
+}

server/migrations/20250530235814-add-collaborator-ids-to-revisions.js

@@ -1,8 +1,8 @@
-'use strict';
+"use strict";
 
 /** @type {import('sequelize-cli').Migration} */
 module.exports = {
-  async up (queryInterface, Sequelize) {
+  async up(queryInterface, Sequelize) {
     await queryInterface.addColumn("revisions", "collaboratorIds", {
       type: Sequelize.ARRAY(Sequelize.UUID),
       allowNull: false,
@@ -10,8 +10,7 @@ module.exports = {
     });
   },
 
-  async down (queryInterface, Sequelize) {
+  async down(queryInterface, Sequelize) {
     await queryInterface.removeColumn("revisions", "collaboratorIds");
-  }
+  },
 };
-

server/queues/tasks/CacheIssueSourcesTask.ts

@@ -3,8 +3,6 @@ import { sequelize } from "@server/storage/database";
 import { Hook, PluginManager } from "@server/utils/PluginManager";
 import BaseTask from "./BaseTask";
 
-const plugins = PluginManager.getHooks(Hook.IssueProvider);
-
 type Props = {
   integrationId: string;
 };
@@ -16,6 +14,7 @@ export default class CacheIssueSourcesTask extends BaseTask<Props> {
       return;
     }
 
+    const plugins = PluginManager.getHooks(Hook.IssueProvider);
     const plugin = plugins.find((p) => p.value.service === integration.service);
     if (!plugin) {
       return;

server/utils/BaseIssueProvider.ts

@@ -12,4 +12,12 @@ export abstract class BaseIssueProvider {
   abstract fetchSources(
     integration: Integration<IntegrationType.Embed>
   ): Promise<IssueSource[]>;
+
+  abstract handleWebhook({
+    payload,
+    headers,
+  }: {
+    payload: Record<string, unknown>;
+    headers: Record<string, unknown>;
+  }): Promise<void>;
 }

yarn.lock

@@ -2886,6 +2886,16 @@
   resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-24.2.0.tgz#3d55c32eac0d38da1a7083a9c3b0cca77924f7d3"
   integrity sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==
 
+"@octokit/openapi-types@^25.0.0":
+  version "25.0.0"
+  resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-25.0.0.tgz#adeead36992abf966e89dcd53518d8b0dc910e0d"
+  integrity sha512-FZvktFu7HfOIJf2BScLKIEYjDsw6RKc7rBJCdvCTfKsVnx2GEB/Nbzjr29DUdb7vQhlzS/j8qDzdditP0OC6aw==
+
+"@octokit/openapi-webhooks-types@10.4.0":
+  version "10.4.0"
+  resolved "https://registry.yarnpkg.com/@octokit/openapi-webhooks-types/-/openapi-webhooks-types-10.4.0.tgz#2cf1e08b9ad9f66930865c9e555499ae2e7507a3"
+  integrity sha512-HMiF7FUiVBYfp8pPijMTkWuPELQB6XkPftrnSuK1C1YXaaq2+0ganiQkorEQfXTmhtwlgHJwXT6P8miVhIyjQA==
+
 "@octokit/plugin-paginate-graphql@^4.0.0":
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-graphql/-/plugin-paginate-graphql-4.0.0.tgz#b26024fa454039c18b948f13bf754ff86b89e8b9"
@@ -2938,6 +2948,13 @@
     deprecation "^2.0.0"
     once "^1.4.0"
 
+"@octokit/request-error@^6.1.7":
+  version "6.1.8"
+  resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-6.1.8.tgz#3c7ce1ca6721eabd43dbddc76b44860de1fdea75"
+  integrity sha512-WEi/R0Jmq+IJKydWlKDmryPcmdYSVjL3ekaiEL1L9eo1sUnqMJ+grqmC9cjk7CA7+b2/T397tO5d8YLOH3qYpQ==
+  dependencies:
+    "@octokit/types" "^14.0.0"
+
 "@octokit/request@^8.0.1", "@octokit/request@^8.0.2", "@octokit/request@^8.3.1":
   version "8.4.1"
   resolved "https://registry.yarnpkg.com/@octokit/request/-/request-8.4.1.tgz#715a015ccf993087977ea4365c44791fc4572486"
@@ -2962,11 +2979,23 @@
   dependencies:
     "@octokit/openapi-types" "^24.2.0"
 
+"@octokit/types@^14.0.0":
+  version "14.0.0"
+  resolved "https://registry.yarnpkg.com/@octokit/types/-/types-14.0.0.tgz#bbd1d31e2269940789ef143b1c37918aae09adc4"
+  integrity sha512-VVmZP0lEhbo2O1pdq63gZFiGCKkm8PPp8AUOijlwPO6hojEVjspA0MWKP7E4hbvGxzFKNqKr6p0IYtOH/Wf/zA==
+  dependencies:
+    "@octokit/openapi-types" "^25.0.0"
+
 "@octokit/webhooks-methods@^4.1.0":
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/@octokit/webhooks-methods/-/webhooks-methods-4.1.0.tgz#681a6c86c9b21d4ec9e29108fb053ae7512be033"
   integrity sha512-zoQyKw8h9STNPqtm28UGOYFE7O6D4Il8VJwhAtMHFt2C4L0VQT1qGKLeefUOqHNs1mNRYSadVv7x0z8U2yyeWQ==
 
+"@octokit/webhooks-methods@^5.1.1":
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/@octokit/webhooks-methods/-/webhooks-methods-5.1.1.tgz#192f11a1f115702833f033293e2fef8f69f612e4"
+  integrity sha512-NGlEHZDseJTCj8TMMFehzwa9g7On4KJMPVHDSrHxCQumL6uSQR8wIkP/qesv52fXqV1BPf4pTxwtS31ldAt9Xg==
+
 "@octokit/webhooks-types@7.6.1":
   version "7.6.1"
   resolved "https://registry.yarnpkg.com/@octokit/webhooks-types/-/webhooks-types-7.6.1.tgz#bc96371057c2d54c982c9f8f642655b26cd588eb"
@@ -2982,6 +3011,15 @@
     "@octokit/webhooks-types" "7.6.1"
     aggregate-error "^3.1.0"
 
+"@octokit/webhooks@^13.8.0":
+  version "13.8.0"
+  resolved "https://registry.yarnpkg.com/@octokit/webhooks/-/webhooks-13.8.0.tgz#0d1f40f5b4217090deb5b810426052f477d8522b"
+  integrity sha512-3PCWyFBNbW2+Ox36VAkSqlPoIb96NZiPcICRYySHZrDTM2NuNxvrjPeaQDj2egqILs9EZFObRTHVMe4XxXJV7w==
+  dependencies:
+    "@octokit/openapi-webhooks-types" "10.4.0"
+    "@octokit/request-error" "^6.1.7"
+    "@octokit/webhooks-methods" "^5.1.1"
+
 "@one-ini/wasm@0.1.1":
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/@one-ini/wasm/-/wasm-0.1.1.tgz#6013659736c9dbfccc96e8a9c2b3de317df39323"