wip

server/scripts/20240810000000-fix-attachments.ts

@@ -0,0 +1,68 @@
+import "./bootstrap";
+import { Op } from "sequelize";
+import { Attachment } from "@server/models";
+import FileStorage from "@server/storage/files";
+
+const limit = 100;
+const page = 0;
+
+export default async function main(exit = false) {
+  const work = async (page: number): Promise<void> => {
+    const teamId = process.argv[2];
+    const dryRun = process.argv[3] !== "execute";
+
+    if (!teamId) {
+      throw new Error("Team ID is required");
+    }
+
+    console.log(`page ${page}, ${dryRun ? "DRY RUN" : ""}`);
+
+    const attachments = await Attachment.unscoped().findAll({
+      attributes: ["id", "key"],
+      where: {
+        teamId,
+        updatedAt: { [Op.gt]: new Date("2024-08-01") },
+        createdAt: { [Op.lt]: new Date("2024-08-01") },
+      },
+      limit,
+      offset: page * limit,
+      order: [["createdAt", "ASC"]],
+      paranoid: false,
+    });
+
+    for (const attachment of attachments) {
+      console.log(`Checking: ${attachment.key}`);
+      const exists = await FileStorage.getFileExists(attachment.key);
+
+      if (!exists) {
+        // key with the last slash in the string replaced with a double slash
+        const possibleKey = attachment.key.replace(/\/([^/]*)$/, "//$1");
+        const existsAlt = await FileStorage.getFileExists(possibleKey);
+        if (existsAlt) {
+          console.log(`Exists at double slash, move it: ${possibleKey}`);
+
+          if (dryRun) {
+            console.log(`Would move: ${possibleKey} to ${attachment.key}`);
+          } else {
+            await FileStorage.moveFile(possibleKey, attachment.key);
+          }
+        }
+      } else {
+        console.log(`Attachment exists: ${attachment.key}`);
+      }
+    }
+
+    return attachments.length === limit ? work(page + 1) : undefined;
+  };
+
+  await work(page);
+
+  if (exit) {
+    console.log("Complete");
+    process.exit(0);
+  }
+}
+
+if (process.env.NODE_ENV !== "test") {
+  void main(true);
+}

server/storage/files/BaseStorage.ts

@@ -216,6 +216,10 @@ export default abstract class BaseStorage {
     }
   }
 
+  public abstract getFileExists(key: string): Promise<boolean>;
+
+  public abstract moveFile(fromKey: string, toKey: string): Promise<void>;
+
   /**
    * Delete a file from the storage provider.
    *

server/storage/files/LocalStorage.ts

@@ -140,6 +140,14 @@ export default class LocalStorage extends BaseStorage {
     return fs.stat(this.getFilePath(key));
   }
 
+  public getFileExists(key: string) {
+    return fs.pathExists(this.getFilePath(key));
+  }
+
+  public moveFile(fromKey: string, toKey: string): Promise<void> {
+    return fs.move(this.getFilePath(fromKey), this.getFilePath(toKey));
+  }
+
   private getFilePath(key: string) {
     invariant(
       env.FILE_STORAGE_LOCAL_ROOT_DIR,

server/storage/files/S3Storage.ts

@@ -5,6 +5,8 @@ import {
   DeleteObjectCommand,
   GetObjectCommand,
   ObjectCannedACL,
+  HeadObjectCommand,
+  CopyObjectCommand,
 } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import "@aws-sdk/signature-v4-crt"; // https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime-crt
@@ -197,6 +199,34 @@ export default class S3Storage extends BaseStorage {
     });
   }
 
+  public getFileExists(key: string): Promise<boolean> {
+    return this.client
+      .send(
+        new HeadObjectCommand({
+          Bucket: this.getBucket(),
+          Key: key,
+        })
+      )
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  public moveFile = async (fromKey: string, toKey: string) => {
+    await this.client.send(
+      new CopyObjectCommand({
+        Bucket: this.getBucket(),
+        CopySource: `${this.getBucket()}/${fromKey}`,
+        Key: toKey,
+      })
+    );
+    await this.client.send(
+      new DeleteObjectCommand({
+        Bucket: this.getBucket(),
+        Key: fromKey,
+      })
+    );
+  };
+
   public getFileStream(
     key: string,
     range?: { start: number; end: number }