From 7d315288dd253820f8858cdcda1e08f6537a1a60 Mon Sep 17 00:00:00 2001 From: Hemachandar <132386067+hmacr@users.noreply.github.com> Date: Wed, 16 Jul 2025 08:37:14 +0530 Subject: [PATCH] Listen to GitHub webhooks to update `issueSources` cache (#9414) * Listen to GitHub webhooks to update issue-sources cache * Add `GitHubWebhookTask` * review --- .env.sample | 1 + package.json | 1 + plugins/github/server/GitHubIssueProvider.ts | 220 +++++++++++++++++- plugins/github/server/api/github.ts | 33 +++ plugins/github/server/env.ts | 6 + plugins/github/server/github.ts | 19 ++ plugins/github/server/index.ts | 5 + .../github/server/tasks/GitHubWebhookTask.ts | 26 +++ server/middlewares/validateWebhook.ts | 36 +++ ...35814-add-collaborator-ids-to-revisions.js | 9 +- server/queues/tasks/CacheIssueSourcesTask.ts | 3 +- server/utils/BaseIssueProvider.ts | 8 + yarn.lock | 38 +++ 13 files changed, 397 insertions(+), 8 deletions(-) create mode 100644 plugins/github/server/tasks/GitHubWebhookTask.ts create mode 100644 server/middlewares/validateWebhook.ts diff --git a/.env.sample b/.env.sample index 49456ae74b..76bb83f560 100644 --- a/.env.sample +++ b/.env.sample @@ -202,6 +202,7 @@ RATE_LIMITER_DURATION_WINDOW=60 # DOCS: https://docs.getoutline.com/s/hosting/doc/github-GchT3NNxI9 GITHUB_CLIENT_ID= GITHUB_CLIENT_SECRET= +GITHUB_WEBHOOK_SECRET= GITHUB_APP_NAME= GITHUB_APP_ID= GITHUB_APP_PRIVATE_KEY= diff --git a/package.json b/package.json index f79bff0984..c07027ca1c 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "@node-oauth/oauth2-server": "^5.2.0", "@notionhq/client": "^2.3.0", "@octokit/auth-app": "^6.1.4", + "@octokit/webhooks": "^13.8.0", "@outlinewiki/koa-passport": "^4.2.1", "@outlinewiki/passport-azure-ad-oauth2": "^0.1.0", "@radix-ui/react-collapsible": "^1.1.11", diff --git a/plugins/github/server/GitHubIssueProvider.ts b/plugins/github/server/GitHubIssueProvider.ts index a8da02f5db..3f59bc2b43 100644 --- a/plugins/github/server/GitHubIssueProvider.ts +++ b/plugins/github/server/GitHubIssueProvider.ts @@ -1,7 +1,14 @@ import { Endpoints } from "@octokit/types"; +import { + InstallationNewPermissionsAcceptedEvent, + InstallationRepositoriesEvent, + RepositoryRenamedEvent, +} from "@octokit/webhooks-types"; import { IssueSource } from "@shared/schema"; import { IntegrationService, IntegrationType } from "@shared/types"; -import { Integration } from "@server/models"; +import Logger from "@server/logging/Logger"; +import { Integration, IntegrationAuthentication } from "@server/models"; +import { sequelize } from "@server/storage/database"; import { BaseIssueProvider } from "@server/utils/BaseIssueProvider"; import { GitHub } from "./github"; @@ -37,4 +44,215 @@ export class GitHubIssueProvider extends BaseIssueProvider { return sources; } + + async handleWebhook({ + payload, + headers, + }: { + payload: Record; + headers: Record; + }) { + const hookId = headers["x-github-hook-id"] as string; + const eventName = headers["x-github-event"] as string; + const action = payload.action as string; + + if (!eventName || !action) { + Logger.warn( + `Received GitHub webhook without event name or action; hookId: ${hookId}, eventName: ${eventName}, action: ${action}` + ); + return; + } + + switch (eventName) { + case "installation": { + await this.handleInstallationEvent(payload, action); + break; + } + + case "installation_repositories": { + await this.handleInstallationRepositoriesEvent( + payload as unknown as InstallationRepositoriesEvent + ); + break; + } + + case "repository": { + await this.handleRepositoryEvent(payload, action, hookId); + break; + } + + default: + } + } + + private async handleInstallationEvent( + payload: Record, + action: string + ): Promise { + if (action !== "new_permissions_accepted") { + return; + } + + const event = payload as unknown as InstallationNewPermissionsAcceptedEvent; + const installationId = event.installation.id; + const integration = await Integration.findOne({ + where: { + service: IntegrationService.GitHub, + "settings.github.installation.id": installationId, + }, + }); + + if (!integration) { + Logger.warn( + `GitHub installation new_permissions_accepted event without integration; installationId: ${installationId}` + ); + return; + } + + const sources = await this.fetchSources(integration); + + const client = await GitHub.authenticateAsInstallation(installationId); + const installation = await client.requestAppInstallation(installationId); + + const scopes = Object.entries(installation.data.permissions).map( + ([name, permission]) => `${name}:${permission}` + ); + + await sequelize.transaction(async (transaction) => { + await integration.reload({ + include: { + model: IntegrationAuthentication, + as: "authentication", + required: true, + }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + const authentication = integration.authentication; + + if (!authentication) { + Logger.warn( + `GitHub integration without authentication; integrationId: ${integration.id}` + ); + return; + } + + authentication.scopes = scopes; + await authentication.save({ transaction }); + + integration.issueSources = sources; + integration.changed("issueSources", true); + await integration.save({ transaction }); + }); + } + + private async handleInstallationRepositoriesEvent( + event: InstallationRepositoriesEvent + ): Promise { + const installationId = event.installation.id; + const account = event.installation.account; + + await sequelize.transaction(async (transaction) => { + const integration = await Integration.findOne({ + where: { + service: IntegrationService.GitHub, + "settings.github.installation.id": installationId, + }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + + if (!integration) { + Logger.warn( + `GitHub installation_repositories event without integration; installationId: ${installationId}` + ); + return; + } + + let sources = integration.issueSources ?? []; + + if (event.action === "added") { + const addedSources = event.repositories_added.map( + (repo) => ({ + id: String(repo.id), + name: repo.name, + owner: { + id: String(account.id), + name: account.login, + }, + service: IntegrationService.GitHub, + }) + ); + sources.push(...addedSources); + } else { + const removedSourceIds = event.repositories_removed.map((repo) => + String(repo.id) + ); + sources = sources.filter( + (source) => !removedSourceIds.includes(source.id) + ); + } + + integration.issueSources = sources; + integration.changed("issueSources", true); + await integration.save({ transaction }); + }); + } + + private async handleRepositoryEvent( + payload: Record, + action: string, + hookId: string + ): Promise { + if (action !== "renamed") { + return; + } + + const event = payload as unknown as RepositoryRenamedEvent; + const installationId = event.installation?.id; + + if (!installationId) { + Logger.warn( + `GitHub repository renamed event without installation ID; hookId: ${hookId}` + ); + return; + } + + const repoId = event.repository.id; + const repoName = event.repository.name; + + await sequelize.transaction(async (transaction) => { + const integration = await Integration.findOne({ + where: { + service: IntegrationService.GitHub, + "settings.github.installation.id": installationId, + }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + + if (!integration) { + Logger.warn( + `GitHub repository renamed event without integration; installationId: ${installationId}` + ); + return; + } + + const source = integration.issueSources?.find( + (s) => s.id === String(repoId) + ); + + if (!source) { + Logger.info( + "task", + `No matching issue source found for repository ID: ${repoId}, integration ID: ${integration.id}` + ); + return; + } + + source.name = repoName; + integration.changed("issueSources", true); + await integration.save({ transaction }); + }); + } } diff --git a/plugins/github/server/api/github.ts b/plugins/github/server/api/github.ts index 96faa2d67d..23412f346e 100644 --- a/plugins/github/server/api/github.ts +++ b/plugins/github/server/api/github.ts @@ -6,10 +6,13 @@ import apexAuthRedirect from "@server/middlewares/apexAuthRedirect"; import auth from "@server/middlewares/authentication"; import { transaction } from "@server/middlewares/transaction"; import validate from "@server/middlewares/validate"; +import validateWebhook from "@server/middlewares/validateWebhook"; import { IntegrationAuthentication, Integration } from "@server/models"; import { APIContext } from "@server/types"; import { GitHubUtils } from "../../shared/GitHubUtils"; +import env from "../env"; import { GitHub } from "../github"; +import GitHubWebhookTask from "../tasks/GitHubWebhookTask"; import * as T from "./schema"; const router = new Router(); @@ -60,11 +63,16 @@ router.get( return ctx.redirect(GitHubUtils.errorUrl("unauthenticated")); } + const scopes = Object.entries(installation.permissions).map( + ([name, permission]) => `${name}:${permission}` + ); + const authentication = await IntegrationAuthentication.create( { service: IntegrationService.GitHub, userId: user.id, teamId: user.teamId, + scopes, }, { transaction } ); @@ -92,4 +100,29 @@ router.get( } ); +router.post( + "github.webhooks", + validateWebhook({ + secretKey: env.GITHUB_WEBHOOK_SECRET!, + getSignatureFromHeader: (ctx) => { + const { headers } = ctx.request; + const signatureHeader = headers["x-hub-signature-256"]; + const signature = Array.isArray(signatureHeader) + ? signatureHeader[0] + : signatureHeader; + return signature?.split("=")[1]; + }, + }), + async (ctx: APIContext) => { + const { headers, body } = ctx.request; + + await new GitHubWebhookTask().schedule({ + payload: body, + headers, + }); + + ctx.status = 202; + } +); + export default router; diff --git a/plugins/github/server/env.ts b/plugins/github/server/env.ts index 80e877e7fb..b1e05d0c30 100644 --- a/plugins/github/server/env.ts +++ b/plugins/github/server/env.ts @@ -26,6 +26,12 @@ class GitHubPluginEnvironment extends Environment { environment.GITHUB_CLIENT_SECRET ); + @IsOptional() + @CannotUseWithout("GITHUB_CLIENT_ID") + public GITHUB_WEBHOOK_SECRET = this.toOptionalString( + environment.GITHUB_WEBHOOK_SECRET + ); + @IsOptional() @CannotUseWithout("GITHUB_APP_PRIVATE_KEY") public GITHUB_APP_ID = this.toOptionalString(environment.GITHUB_APP_ID); diff --git a/plugins/github/server/github.ts b/plugins/github/server/github.ts index a8ef1ce9bc..213625fc05 100644 --- a/plugins/github/server/github.ts +++ b/plugins/github/server/github.ts @@ -22,6 +22,8 @@ type PR = Endpoints["GET /repos/{owner}/{repo}/pulls/{pull_number}"]["response"]["data"]; type Issue = Endpoints["GET /repos/{owner}/{repo}/issues/{issue_number}"]["response"]["data"]; +type Installation = + Endpoints["GET /app/installations/{installation_id}"]["response"]["data"]; const requestPlugin = (octokit: Octokit) => ({ requestRepos: () => @@ -86,6 +88,23 @@ const requestPlugin = (octokit: Octokit) => ({ } }, + /** + * Fetches details of a specific GitHub app installation + * + * @param installationId Id of the installation to fetch + * @returns Response containing installation details + */ + requestAppInstallation: async ( + installationId: number + ): Promise> => + octokit.request("GET /app/installations/{installation_id}", { + installation_id: installationId, + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + }), + /** * Uninstalls the GitHub app from a given target * diff --git a/plugins/github/server/index.ts b/plugins/github/server/index.ts index b56dd83a9d..1c47fbac26 100644 --- a/plugins/github/server/index.ts +++ b/plugins/github/server/index.ts @@ -5,6 +5,7 @@ import { GitHubIssueProvider } from "./GitHubIssueProvider"; import router from "./api/github"; import env from "./env"; import { GitHub } from "./github"; +import GitHubWebhookTask from "./tasks/GitHubWebhookTask"; import { uninstall } from "./uninstall"; const enabled = @@ -21,6 +22,10 @@ if (enabled) { type: Hook.API, value: router, }, + { + type: Hook.Task, + value: GitHubWebhookTask, + }, { type: Hook.IssueProvider, value: new GitHubIssueProvider(), diff --git a/plugins/github/server/tasks/GitHubWebhookTask.ts b/plugins/github/server/tasks/GitHubWebhookTask.ts new file mode 100644 index 0000000000..61e710f147 --- /dev/null +++ b/plugins/github/server/tasks/GitHubWebhookTask.ts @@ -0,0 +1,26 @@ +import { IntegrationService } from "@shared/types"; +import BaseTask from "@server/queues/tasks/BaseTask"; +import { Hook, PluginManager } from "@server/utils/PluginManager"; + +type Props = { + headers: Record; + payload: Record; +}; + +export default class GitHubWebhookTask extends BaseTask { + public async perform({ headers, payload }: Props): Promise { + const plugins = PluginManager.getHooks(Hook.IssueProvider); + const plugin = plugins.find( + (p) => p.value.service === IntegrationService.GitHub + ); + + if (!plugin) { + return; + } + + await plugin.value.handleWebhook({ + headers, + payload, + }); + } +} diff --git a/server/middlewares/validateWebhook.ts b/server/middlewares/validateWebhook.ts new file mode 100644 index 0000000000..c237567d1d --- /dev/null +++ b/server/middlewares/validateWebhook.ts @@ -0,0 +1,36 @@ +import crypto from "crypto"; +import { Next } from "koa"; +import { APIContext } from "@server/types"; +import { safeEqual } from "@server/utils/crypto"; + +export default function validateWebhook({ + secretKey, + getSignatureFromHeader, +}: { + secretKey: string; + getSignatureFromHeader: (ctx: APIContext) => string | undefined; +}) { + return async function validateWebhookMiddleware(ctx: APIContext, next: Next) { + const { body } = ctx.request; + const signatureFromHeader = getSignatureFromHeader(ctx); + + if (!signatureFromHeader) { + ctx.status = 401; + ctx.body = "Missing signature header"; + return; + } + + const computedSignature = crypto + .createHmac("sha256", secretKey) + .update(JSON.stringify(body)) + .digest("hex"); + + if (!safeEqual(computedSignature, signatureFromHeader)) { + ctx.status = 401; + ctx.body = "Invalid signature"; + return; + } + + return next(); + }; +} diff --git a/server/migrations/20250530235814-add-collaborator-ids-to-revisions.js b/server/migrations/20250530235814-add-collaborator-ids-to-revisions.js index 1bffcffac6..3feb31ff94 100644 --- a/server/migrations/20250530235814-add-collaborator-ids-to-revisions.js +++ b/server/migrations/20250530235814-add-collaborator-ids-to-revisions.js @@ -1,8 +1,8 @@ -'use strict'; +"use strict"; /** @type {import('sequelize-cli').Migration} */ module.exports = { - async up (queryInterface, Sequelize) { + async up(queryInterface, Sequelize) { await queryInterface.addColumn("revisions", "collaboratorIds", { type: Sequelize.ARRAY(Sequelize.UUID), allowNull: false, @@ -10,8 +10,7 @@ module.exports = { }); }, - async down (queryInterface, Sequelize) { + async down(queryInterface, Sequelize) { await queryInterface.removeColumn("revisions", "collaboratorIds"); - } + }, }; - diff --git a/server/queues/tasks/CacheIssueSourcesTask.ts b/server/queues/tasks/CacheIssueSourcesTask.ts index 053e98d44c..b824a9f3b6 100644 --- a/server/queues/tasks/CacheIssueSourcesTask.ts +++ b/server/queues/tasks/CacheIssueSourcesTask.ts @@ -3,8 +3,6 @@ import { sequelize } from "@server/storage/database"; import { Hook, PluginManager } from "@server/utils/PluginManager"; import BaseTask from "./BaseTask"; -const plugins = PluginManager.getHooks(Hook.IssueProvider); - type Props = { integrationId: string; }; @@ -16,6 +14,7 @@ export default class CacheIssueSourcesTask extends BaseTask { return; } + const plugins = PluginManager.getHooks(Hook.IssueProvider); const plugin = plugins.find((p) => p.value.service === integration.service); if (!plugin) { return; diff --git a/server/utils/BaseIssueProvider.ts b/server/utils/BaseIssueProvider.ts index 6a6b961663..79ebe8cda6 100644 --- a/server/utils/BaseIssueProvider.ts +++ b/server/utils/BaseIssueProvider.ts @@ -12,4 +12,12 @@ export abstract class BaseIssueProvider { abstract fetchSources( integration: Integration ): Promise; + + abstract handleWebhook({ + payload, + headers, + }: { + payload: Record; + headers: Record; + }): Promise; } diff --git a/yarn.lock b/yarn.lock index 8d38a94469..d99f098c09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2886,6 +2886,16 @@ resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-24.2.0.tgz#3d55c32eac0d38da1a7083a9c3b0cca77924f7d3" integrity sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg== +"@octokit/openapi-types@^25.0.0": + version "25.0.0" + resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-25.0.0.tgz#adeead36992abf966e89dcd53518d8b0dc910e0d" + integrity sha512-FZvktFu7HfOIJf2BScLKIEYjDsw6RKc7rBJCdvCTfKsVnx2GEB/Nbzjr29DUdb7vQhlzS/j8qDzdditP0OC6aw== + +"@octokit/openapi-webhooks-types@10.4.0": + version "10.4.0" + resolved "https://registry.yarnpkg.com/@octokit/openapi-webhooks-types/-/openapi-webhooks-types-10.4.0.tgz#2cf1e08b9ad9f66930865c9e555499ae2e7507a3" + integrity sha512-HMiF7FUiVBYfp8pPijMTkWuPELQB6XkPftrnSuK1C1YXaaq2+0ganiQkorEQfXTmhtwlgHJwXT6P8miVhIyjQA== + "@octokit/plugin-paginate-graphql@^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-graphql/-/plugin-paginate-graphql-4.0.0.tgz#b26024fa454039c18b948f13bf754ff86b89e8b9" @@ -2938,6 +2948,13 @@ deprecation "^2.0.0" once "^1.4.0" +"@octokit/request-error@^6.1.7": + version "6.1.8" + resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-6.1.8.tgz#3c7ce1ca6721eabd43dbddc76b44860de1fdea75" + integrity sha512-WEi/R0Jmq+IJKydWlKDmryPcmdYSVjL3ekaiEL1L9eo1sUnqMJ+grqmC9cjk7CA7+b2/T397tO5d8YLOH3qYpQ== + dependencies: + "@octokit/types" "^14.0.0" + "@octokit/request@^8.0.1", "@octokit/request@^8.0.2", "@octokit/request@^8.3.1": version "8.4.1" resolved "https://registry.yarnpkg.com/@octokit/request/-/request-8.4.1.tgz#715a015ccf993087977ea4365c44791fc4572486" @@ -2962,11 +2979,23 @@ dependencies: "@octokit/openapi-types" "^24.2.0" +"@octokit/types@^14.0.0": + version "14.0.0" + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-14.0.0.tgz#bbd1d31e2269940789ef143b1c37918aae09adc4" + integrity sha512-VVmZP0lEhbo2O1pdq63gZFiGCKkm8PPp8AUOijlwPO6hojEVjspA0MWKP7E4hbvGxzFKNqKr6p0IYtOH/Wf/zA== + dependencies: + "@octokit/openapi-types" "^25.0.0" + "@octokit/webhooks-methods@^4.1.0": version "4.1.0" resolved "https://registry.yarnpkg.com/@octokit/webhooks-methods/-/webhooks-methods-4.1.0.tgz#681a6c86c9b21d4ec9e29108fb053ae7512be033" integrity sha512-zoQyKw8h9STNPqtm28UGOYFE7O6D4Il8VJwhAtMHFt2C4L0VQT1qGKLeefUOqHNs1mNRYSadVv7x0z8U2yyeWQ== +"@octokit/webhooks-methods@^5.1.1": + version "5.1.1" + resolved "https://registry.yarnpkg.com/@octokit/webhooks-methods/-/webhooks-methods-5.1.1.tgz#192f11a1f115702833f033293e2fef8f69f612e4" + integrity sha512-NGlEHZDseJTCj8TMMFehzwa9g7On4KJMPVHDSrHxCQumL6uSQR8wIkP/qesv52fXqV1BPf4pTxwtS31ldAt9Xg== + "@octokit/webhooks-types@7.6.1": version "7.6.1" resolved "https://registry.yarnpkg.com/@octokit/webhooks-types/-/webhooks-types-7.6.1.tgz#bc96371057c2d54c982c9f8f642655b26cd588eb" @@ -2982,6 +3011,15 @@ "@octokit/webhooks-types" "7.6.1" aggregate-error "^3.1.0" +"@octokit/webhooks@^13.8.0": + version "13.8.0" + resolved "https://registry.yarnpkg.com/@octokit/webhooks/-/webhooks-13.8.0.tgz#0d1f40f5b4217090deb5b810426052f477d8522b" + integrity sha512-3PCWyFBNbW2+Ox36VAkSqlPoIb96NZiPcICRYySHZrDTM2NuNxvrjPeaQDj2egqILs9EZFObRTHVMe4XxXJV7w== + dependencies: + "@octokit/openapi-webhooks-types" "10.4.0" + "@octokit/request-error" "^6.1.7" + "@octokit/webhooks-methods" "^5.1.1" + "@one-ini/wasm@0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@one-ini/wasm/-/wasm-0.1.1.tgz#6013659736c9dbfccc96e8a9c2b3de317df39323"