Merge branch 'main' into hmacr/8506-cache-issue-sources

plugins/github/server/GitHubIssueProvider.ts

@@ -0,0 +1,40 @@
+import { Endpoints } from "@octokit/types";
+import { IssueSource } from "@shared/schema";
+import { IntegrationService, IntegrationType } from "@shared/types";
+import { Integration } from "@server/models";
+import { BaseIssueProvider } from "@server/utils/BaseIssueProvider";
+import { GitHub } from "./github";
+
+// This is needed to handle Octokit paginate response type mismatch.
+type ReposForInstallation =
+  Endpoints["GET /installation/repositories"]["response"]["data"]["repositories"];
+
+export class GitHubIssueProvider extends BaseIssueProvider {
+  constructor() {
+    super(IntegrationService.GitHub);
+  }
+
+  async fetchSources(
+    integration: Integration<IntegrationType.Embed>
+  ): Promise<IssueSource[]> {
+    const client = await GitHub.authenticateAsInstallation(
+      integration.settings.github!.installation.id
+    );
+
+    const sources: IssueSource[] = [];
+
+    for await (const response of client.requestRepos()) {
+      const repos = response.data as unknown as ReposForInstallation;
+      sources.push(
+        ...repos.map<IssueSource>((repo) => ({
+          id: String(repo.id),
+          name: repo.name,
+          owner: { id: String(repo.owner.id), name: repo.owner.login },
+          service: IntegrationService.GitHub,
+        }))
+      );
+    }
+
+    return sources;
+  }
+}

plugins/github/server/api/github.ts

@@ -2,6 +2,7 @@ import Router from "koa-router";
 import find from "lodash/find";
 import { IntegrationService, IntegrationType } from "@shared/types";
 import { parseDomain } from "@shared/utils/domains";
+import { createContext } from "@server/context";
 import Logger from "@server/logging/Logger";
 import auth from "@server/middlewares/authentication";
 import { transaction } from "@server/middlewares/transaction";
@@ -88,30 +89,27 @@ router.get(
       },
       { transaction }
     );
-    await Integration.create(
-      {
-        service: IntegrationService.GitHub,
-        type: IntegrationType.Embed,
-        userId: user.id,
-        teamId: user.teamId,
-        authenticationId: authentication.id,
-        settings: {
-          github: {
-            installation: {
-              id: installationId!,
-              account: {
-                id: installation.account?.id,
-                name:
-                  // @ts-expect-error Property 'login' does not exist on type
-                  installation.account?.login,
-                avatarUrl: installation.account?.avatar_url,
-              },
+    await Integration.createWithCtx(createContext({ user, transaction }), {
+      service: IntegrationService.GitHub,
+      type: IntegrationType.Embed,
+      userId: user.id,
+      teamId: user.teamId,
+      authenticationId: authentication.id,
+      settings: {
+        github: {
+          installation: {
+            id: installationId!,
+            account: {
+              id: installation.account?.id,
+              name:
+                // @ts-expect-error Property 'login' does not exist on type
+                installation.account?.login,
+              avatarUrl: installation.account?.avatar_url,
             },
           },
         },
       },
-      { transaction }
-    );
+    });
     ctx.redirect(GitHubUtils.url);
   }
 );

plugins/github/server/github.ts

@@ -24,6 +24,17 @@ type Issue =
   Endpoints["GET /repos/{owner}/{repo}/issues/{issue_number}"]["response"]["data"];
 
 const requestPlugin = (octokit: Octokit) => ({
+  requestRepos: () =>
+    octokit.paginate.iterator(
+      octokit.rest.apps.listReposAccessibleToInstallation,
+      {
+        headers: {
+          Accept: "application/vnd.github+json",
+          "X-GitHub-Api-Version": "2022-11-28",
+        },
+      }
+    ),
+
   requestPR: async (params: NonNullable<ReturnType<typeof GitHub.parseUrl>>) =>
     octokit.request(`GET /repos/{owner}/{repo}/pulls/{pull_number}`, {
       owner: params.owner,

plugins/github/server/index.ts

@@ -1,6 +1,7 @@
 import { Minute } from "@shared/utils/time";
 import { PluginManager, Hook } from "@server/utils/PluginManager";
 import config from "../plugin.json";
+import { GitHubIssueProvider } from "./GitHubIssueProvider";
 import router from "./api/github";
 import env from "./env";
 import { GitHub } from "./github";
@@ -20,6 +21,10 @@ if (enabled) {
       type: Hook.API,
       value: router,
     },
+    {
+      type: Hook.IssueProvider,
+      value: new GitHubIssueProvider(),
+    },
     {
       type: Hook.UnfurlProvider,
       value: { unfurl: GitHub.unfurl, cacheExpiry: Minute.seconds },

server/migrations/20250409184249-add-issueSources-to-integrations.js

@@ -0,0 +1,15 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    queryInterface.addColumn("integrations", "issueSources", {
+      type: Sequelize.JSONB,
+      allowNull: true,
+    });
+  },
+
+  async down(queryInterface, Sequelize) {
+    queryInterface.removeColumn("integrations", "issueSources");
+  },
+};

server/models/Integration.ts

@@ -13,6 +13,7 @@ import {
   IsIn,
   AfterDestroy,
 } from "sequelize-typescript";
+import { IssueSource } from "@shared/schema";
 import { IntegrationType, IntegrationService } from "@shared/types";
 import type { IntegrationSettings } from "@shared/types";
 import Collection from "@server/models/Collection";
@@ -53,6 +54,9 @@ class Integration<T = unknown> extends ParanoidModel<
   @Column(DataType.ARRAY(DataType.STRING))
   events: string[];
 
+  @Column(DataType.JSONB)
+  issueSources: IssueSource[] | null;
+
   // associations
 
   @BelongsTo(() => User, "userId")

server/queues/processors/IntegrationCreatedProcessor.ts

@@ -3,6 +3,7 @@ import { Integration } from "@server/models";
 import BaseProcessor from "@server/queues/processors/BaseProcessor";
 import { IntegrationEvent, Event } from "@server/types";
 import { CacheHelper } from "@server/utils/CacheHelper";
+import CacheIssueSourcesTask from "../tasks/CacheIssueSourcesTask";
 
 export default class IntegrationCreatedProcessor extends BaseProcessor {
   static applicableEvents: Event["name"][] = ["integrations.create"];
@@ -18,6 +19,11 @@ export default class IntegrationCreatedProcessor extends BaseProcessor {
       return;
     }
 
+    // Store the available issue sources in the integration record.
+    await CacheIssueSourcesTask.schedule({
+      integrationId: integration.id,
+    });
+
     // Clear the cache of unfurled data for the team as it may be stale now.
     await CacheHelper.clearData(CacheHelper.getUnfurlKey(integration.teamId));
   }

server/queues/tasks/CacheIssueSourcesTask.ts

@@ -0,0 +1,32 @@
+import { Integration } from "@server/models";
+import { sequelize } from "@server/storage/database";
+import { Hook, PluginManager } from "@server/utils/PluginManager";
+import BaseTask from "./BaseTask";
+
+const plugins = PluginManager.getHooks(Hook.IssueProvider);
+
+type Props = {
+  integrationId: string;
+};
+
+export default class CacheIssueSourcesTask extends BaseTask<Props> {
+  async perform({ integrationId }: Props) {
+    const integration = await Integration.findByPk(integrationId);
+    if (!integration) {
+      return;
+    }
+
+    const plugin = plugins.find((p) => p.value.service === integration.service);
+    if (!plugin) {
+      return;
+    }
+
+    const sources = await plugin.value.fetchSources(integration);
+
+    await sequelize.transaction(async (transaction) => {
+      await integration.reload({ transaction, lock: transaction.LOCK.UPDATE });
+      integration.issueSources = sources;
+      await integration.save({ transaction });
+    });
+  }
+}

server/utils/BaseIssueProvider.ts

@@ -0,0 +1,15 @@
+import { IssueSource } from "@shared/schema";
+import { IntegrationType, IssueTrackerIntegrationService } from "@shared/types";
+import { Integration } from "@server/models";
+
+export abstract class BaseIssueProvider {
+  service: IssueTrackerIntegrationService;
+
+  constructor(service: IssueTrackerIntegrationService) {
+    this.service = service;
+  }
+
+  abstract fetchSources(
+    integration: Integration<IntegrationType.Embed>
+  ): Promise<IssueSource[]>;
+}

server/utils/PluginManager.ts

@@ -9,6 +9,7 @@ import Logger from "@server/logging/Logger";
 import type BaseProcessor from "@server/queues/processors/BaseProcessor";
 import type BaseTask from "@server/queues/tasks/BaseTask";
 import { UnfurlSignature, UninstallSignature } from "@server/types";
+import { BaseIssueProvider } from "./BaseIssueProvider";
 
 export enum PluginPriority {
   VeryHigh = 0,
@@ -25,6 +26,7 @@ export enum Hook {
   API = "api",
   AuthProvider = "authProvider",
   EmailTemplate = "emailTemplate",
+  IssueProvider = "issueProvider",
   Processor = "processor",
   Task = "task",
   UnfurlProvider = "unfurl",
@@ -39,6 +41,7 @@ type PluginValueMap = {
   [Hook.API]: Router;
   [Hook.AuthProvider]: { router: Router; id: string };
   [Hook.EmailTemplate]: typeof BaseEmail;
+  [Hook.IssueProvider]: BaseIssueProvider;
   [Hook.Processor]: typeof BaseProcessor;
   [Hook.Task]: typeof BaseTask<any>;
   [Hook.Uninstall]: UninstallSignature;

shared/schema.ts

@@ -3,6 +3,7 @@ import {
   CollectionPermission,
   type ImportableIntegrationService,
   IntegrationService,
+  IssueTrackerIntegrationService,
   ProsemirrorDoc,
 } from "./types";
 import { PageType } from "plugins/notion/shared/types";
@@ -57,3 +58,15 @@ export type ImportTaskOutput = {
   createdAt?: Date;
   updatedAt?: Date;
 }[];
+
+export const IssueSource = z.object({
+  id: z.string().nonempty(),
+  name: z.string().nonempty(),
+  owner: z.object({
+    id: z.string().nonempty(),
+    name: z.string().nonempty(),
+  }),
+  service: z.nativeEnum(IssueTrackerIntegrationService),
+});
+
+export type IssueSource = z.infer<typeof IssueSource>;

shared/types.ts

@@ -129,6 +129,15 @@ export const ImportableIntegrationService = {
   Notion: IntegrationService.Notion,
 } as const;
 
+export type IssueTrackerIntegrationService = Extract<
+  IntegrationService,
+  IntegrationService.GitHub
+>;
+
+export const IssueTrackerIntegrationService = {
+  GitHub: IntegrationService.GitHub,
+} as const;
+
 export type UserCreatableIntegrationService = Extract<
   IntegrationService,
   | IntegrationService.Diagrams