GitHub/outline
1
0
Fork 0
mirror of https://github.com/outline/outline.git synced 2026-06-13 03:14:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

chore: Migrate from JSZip to Yazl (#12408)

Browse Source 
* chore: Migrate from JSZip to Yazl

* Add koa stream helper, PR feedback
This commit is contained in:
Tom Moor
2026-05-21 23:27:23 -04:00
committed by GitHub
parent d43280f08e
commit 8d44a0fd92
13 changed files with 213 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+2 -1
View File
@@ -152,7 +152,6 @@
"js-yaml": "^4.1.1",
"jsdom": "^22.1.0",
"jsonwebtoken": "^9.0.3",
"jszip": "^3.10.1",
"katex": "^0.16.45",
"kbar": "0.1.0-beta.48",
"koa": "^3.0.3",
@@ -271,6 +270,7 @@
"y-prosemirror": "^1.3.7",
"y-protocols": "^1.0.7",
"yauzl": "^3.2.1",
"yazl": "^3.3.1",
"yjs": "^13.6.30",
"zod": "^4.3.6"
},
@@ -348,6 +348,7 @@
"@types/utf8": "^3.0.3",
"@types/validator": "^13.15.10",
"@types/yauzl": "^2.10.3",
"@types/yazl": "^2.4.6",
"@vitest/ui": "^4.1.6",
"babel-plugin-module-resolver": "^5.0.3",
"babel-plugin-styled-components": "^2.1.4",
server/queues/tasks/ExportDocumentTreeTask.ts
+34 -40
View File
@@ -1,6 +1,6 @@
import path from "node:path";
import type JSZip from "jszip";
import { escapeRegExp } from "es-toolkit/compat";
import type { ZipFile } from "yazl";
import type { NavigationNode } from "@shared/types";
import { FileOperationFormat } from "@shared/types";
import Logger from "@server/logging/Logger";
@@ -17,7 +17,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
/**
* Exports the document tree to the given zip instance.
*
* @param zip The JSZip instance to add files to
* @param zip The yazl ZipFile to add files to
* @param documentId The document ID to export
* @param pathInZip The path in the zip to add the document to
* @param format The format to export in
@@ -30,7 +30,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
includeAttachments,
pathMap,
}: {
zip: JSZip;
zip: ZipFile;
pathInZip: string;
documentId: string;
format: FileOperationFormat;
@@ -64,38 +64,33 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
// Add any referenced attachments to the zip file and replace the
// reference in the document with the path to the attachment in the zip
await Promise.all(
attachments.map(async (attachment) => {
Logger.debug("task", `Adding attachment to archive`, {
documentId,
key: attachment.key,
for (const attachment of attachments) {
Logger.debug("task", `Adding attachment to archive`, {
documentId,
key: attachment.key,
});
const dir = path.dirname(pathInZip);
let buffer: Buffer;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
buffer = Buffer.from("");
}
zip.addBuffer(buffer, path.join(dir, attachment.key), {
mtime: attachment.updatedAt,
});
const dir = path.dirname(pathInZip);
zip.file(
path.join(dir, attachment.key),
new Promise<Buffer>((resolve) => {
attachment.buffer.then(resolve).catch((err) => {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
resolve(Buffer.from(""));
});
}),
{
date: attachment.updatedAt,
createFolders: true,
}
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
encodeURI(attachment.key)
);
})
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
encodeURI(attachment.key)
);
}
// Replace any internal links with relative paths to the document in the zip
const internalLinks = [
@@ -117,10 +112,9 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
});
// Finally, add the document to the zip file
zip.file(pathInZip, text, {
date: document.updatedAt,
createFolders: true,
comment: JSON.stringify({
zip.addBuffer(Buffer.from(text), pathInZip, {
mtime: document.updatedAt,
fileComment: JSON.stringify({
createdAt: document.createdAt,
updatedAt: document.updatedAt,
}),
@@ -131,7 +125,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
* Exports the documents and attachments in the given collections to a zip file
* and returns the path to the zip file in tmp.
*
* @param zip The JSZip instance to add files to
* @param zip The yazl ZipFile to add files to
* @param collections The collections to export
* @param format The format to export in
* @param includeAttachments Whether to include attachments in the export
@@ -139,7 +133,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
* @returns The path to the zip file in tmp.
*/
protected async addCollectionsToArchive(
zip: JSZip,
zip: ZipFile,
collections: Collection[],
format: FileOperationFormat,
includeAttachments = true
@@ -178,7 +172,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
document: Document;
format: FileOperationFormat;
documentStructure: NavigationNode[];
zip: JSZip;
zip: ZipFile;
}) {
const pathMap = new Map<string, string>();
server/queues/tasks/ExportHTMLZipTask.ts
+3 -3
View File
@@ -1,4 +1,4 @@
import JSZip from "jszip";
import { ZipFile } from "yazl";
import type { NavigationNode } from "@shared/types";
import { FileOperationFormat } from "@shared/types";
import type { Collection, FileOperation } from "@server/models";
@@ -10,7 +10,7 @@ export default class ExportHTMLZipTask extends ExportDocumentTreeTask {
collections: Collection[],
fileOperation: FileOperation
) {
const zip = new JSZip();
const zip = new ZipFile();
return await this.addCollectionsToArchive(
zip,
@@ -24,7 +24,7 @@ export default class ExportHTMLZipTask extends ExportDocumentTreeTask {
document: Document,
documentStructure: NavigationNode[]
): Promise<string> {
const zip = new JSZip();
const zip = new ZipFile();
return await this.addDocumentToArchive({
document,
server/queues/tasks/ExportJSONTask.ts
+41 -39
View File
@@ -1,4 +1,4 @@
import JSZip from "jszip";
import { ZipFile } from "yazl";
import { omit } from "es-toolkit/compat";
import type { NavigationNode } from "@shared/types";
import env from "@server/env";
@@ -19,7 +19,7 @@ export default class ExportJSONTask extends ExportTask {
collections: Collection[],
fileOperation: FileOperation
) {
const zip = new JSZip();
const zip = new ZipFile();
const usedFilenames = new Set<string>();
// serial to avoid overloading, slow and steady wins the race
@@ -44,7 +44,10 @@ export default class ExportJSONTask extends ExportTask {
return ZipHelper.toTmpFile(zip);
}
private async addMetadataToArchive(zip: JSZip, fileOperation: FileOperation) {
private async addMetadataToArchive(
zip: ZipFile,
fileOperation: FileOperation
) {
const user = await fileOperation.$get("user");
const metadata: JSONExportMetadata = {
@@ -55,16 +58,18 @@ export default class ExportJSONTask extends ExportTask {
createdByEmail: user?.email ?? null,
};
zip.file(
`metadata.json`,
env.isDevelopment
? JSON.stringify(metadata, null, 2)
: JSON.stringify(metadata)
zip.addBuffer(
Buffer.from(
env.isDevelopment
? JSON.stringify(metadata, null, 2)
: JSON.stringify(metadata)
),
`metadata.json`
);
}
private async addCollectionToArchive(
zip: JSZip,
zip: ZipFile,
collection: Collection,
includeAttachments: boolean,
filename: string
@@ -82,32 +87,27 @@ export default class ExportJSONTask extends ExportTask {
};
async function addAttachments(attachments: Attachment[]) {
await Promise.all(
attachments.map(async (attachment) => {
zip.file(
attachment.key,
new Promise<Buffer>((resolve) => {
attachment.buffer.then(resolve).catch((err) => {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
resolve(Buffer.from(""));
});
}),
{
date: attachment.updatedAt,
createFolders: true,
}
);
for (const attachment of attachments) {
let buffer: Buffer;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
buffer = Buffer.from("");
}
zip.addBuffer(buffer, attachment.key, {
mtime: attachment.updatedAt,
});
output.attachments[attachment.id] = {
...omit(presentAttachment(attachment), "url"),
key: attachment.key,
};
})
);
output.attachments[attachment.id] = {
...omit(presentAttachment(attachment), "url"),
key: attachment.key,
};
}
}
async function addDocumentTree(nodes: NavigationNode[]) {
@@ -175,11 +175,13 @@ export default class ExportJSONTask extends ExportTask {
await addDocumentTree(collection.documentStructure);
}
zip.file(
`${filename}.json`,
env.isDevelopment
? JSON.stringify(output, null, 2)
: JSON.stringify(output)
zip.addBuffer(
Buffer.from(
env.isDevelopment
? JSON.stringify(output, null, 2)
: JSON.stringify(output)
),
`${filename}.json`
);
}
server/queues/tasks/ExportMarkdownZipTask.ts
+3 -3
View File
@@ -1,4 +1,4 @@
import JSZip from "jszip";
import { ZipFile } from "yazl";
import type { NavigationNode } from "@shared/types";
import { FileOperationFormat } from "@shared/types";
import type { Collection, FileOperation } from "@server/models";
@@ -10,7 +10,7 @@ export default class ExportMarkdownZipTask extends ExportDocumentTreeTask {
collections: Collection[],
fileOperation: FileOperation
) {
const zip = new JSZip();
const zip = new ZipFile();
return await this.addCollectionsToArchive(
zip,
@@ -24,7 +24,7 @@ export default class ExportMarkdownZipTask extends ExportDocumentTreeTask {
document: Document,
documentStructure: NavigationNode[]
): Promise<string> {
const zip = new JSZip();
const zip = new ZipFile();
return await this.addDocumentToArchive({
document,
server/routes/api/documents/documents.ts
+19 -36
View File
@@ -3,7 +3,6 @@ import fractionalIndex from "fractional-index";
import fs from "fs-extra";
import invariant from "invariant";
import contentDisposition from "content-disposition";
import JSZip from "jszip";
import Router from "koa-router";
import { escapeRegExp, has, remove, uniq } from "es-toolkit/compat";
import mime from "mime-types";
@@ -84,8 +83,8 @@ import EmptyTrashTask from "@server/queues/tasks/EmptyTrashTask";
import FileStorage from "@server/storage/files";
import type { APIContext } from "@server/types";
import { RateLimiterStrategy } from "@server/utils/RateLimiter";
import ZipHelper from "@server/utils/ZipHelper";
import { convertBareUrlsToEmbedMarkdown } from "@server/utils/embeds";
import { streamZipResponse } from "@server/utils/koa";
import { getTeamFromContext } from "@server/utils/passport";
import { assertPresent } from "@server/validation";
import pagination, { paginateQuery } from "../middlewares/pagination";
@@ -890,51 +889,35 @@ router.post(
return;
}
const zip = new JSZip();
await Promise.all(
attachments.map(async (attachment) => {
await streamZipResponse(ctx, `${fileName}.zip`, async (zip) => {
for (const attachment of attachments) {
const location = path.join(
"attachments",
`${attachment.id}.${mime.extension(attachment.contentType)}`
);
zip.file(
location,
new Promise<Buffer>((resolve) => {
attachment.buffer.then(resolve).catch((err) => {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
resolve(Buffer.from(""));
});
}),
{
date: attachment.updatedAt,
createFolders: true,
}
);
let buffer: Buffer;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
buffer = Buffer.from("");
}
zip.addBuffer(buffer, location, { mtime: attachment.updatedAt });
content = content.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
location
);
})
);
}
zip.file(`${fileName}.${extension}`, content, {
date: document.updatedAt,
zip.addBuffer(Buffer.from(content), `${fileName}.${extension}`, {
mtime: document.updatedAt,
});
});
ctx.set("Content-Type", "application/zip");
ctx.set(
"Content-Disposition",
contentDisposition(`${fileName}.zip`, {
type: "attachment",
})
);
ctx.body = zip.generateNodeStream(ZipHelper.defaultStreamOptions);
}
);
server/routes/api/middlewares/apiResponse.ts
+1 -4
View File
@@ -11,10 +11,7 @@ export default function apiResponse() {
typeof ctx.body === "object" &&
!(ctx.body instanceof Readable) &&
!(ctx.body instanceof stream.Readable) &&
!(ctx.body instanceof Buffer) &&
// JSZip returns a wrapped stream instance that is not a true readable stream
// and not exported from the module either, so we must identify it like so.
!(ctx.body && "_readableState" in ctx.body)
!(ctx.body instanceof Buffer)
) {
ctx.body = {
...ctx.body,
server/routes/api/revisions/revisions.ts
+19 -36
View File
@@ -1,7 +1,6 @@
import path from "node:path";
import Router from "koa-router";
import contentDisposition from "content-disposition";
import JSZip from "jszip";
import { escapeRegExp } from "es-toolkit/compat";
import mime from "mime-types";
import { UserRole } from "@shared/types";
@@ -20,7 +19,7 @@ import { authorize } from "@server/policies";
import { presentPolicies, presentRevision } from "@server/presenters";
import type { APIContext } from "@server/types";
import { RateLimiterStrategy } from "@server/utils/RateLimiter";
import ZipHelper from "@server/utils/ZipHelper";
import { streamZipResponse } from "@server/utils/koa";
import pagination from "../middlewares/pagination";
import * as T from "./schema";
@@ -195,51 +194,35 @@ router.post(
return;
}
const zip = new JSZip();
await Promise.all(
attachments.map(async (attachment) => {
await streamZipResponse(ctx, `${fileName}.zip`, async (zip) => {
for (const attachment of attachments) {
const location = path.join(
"attachments",
`${attachment.id}.${mime.extension(attachment.contentType)}`
);
zip.file(
location,
new Promise<Buffer>((resolve) => {
attachment.buffer.then(resolve).catch((err) => {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
resolve(Buffer.from(""));
});
}),
{
date: attachment.updatedAt,
createFolders: true,
}
);
let buffer: Buffer;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
buffer = Buffer.from("");
}
zip.addBuffer(buffer, location, { mtime: attachment.updatedAt });
content = content.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
location
);
})
);
}
zip.file(`${fileName}.${extension}`, content, {
date: revision.updatedAt,
zip.addBuffer(Buffer.from(content), `${fileName}.${extension}`, {
mtime: revision.updatedAt,
});
});
ctx.set("Content-Type", "application/zip");
ctx.set(
"Content-Disposition",
contentDisposition(`${fileName}.zip`, {
type: "attachment",
})
);
ctx.body = zip.generateNodeStream(ZipHelper.defaultStreamOptions);
}
);
server/typings/yazl.d.ts
Vendored
+7
View File
@@ -0,0 +1,7 @@
import "yazl";
declare module "yazl" {
interface Options {
fileComment: string | Buffer;
}
}
server/utils/ZipHelper.test.ts
+11 -5
View File
@@ -1,20 +1,26 @@
import path from "node:path";
import fs from "fs-extra";
import JSZip from "jszip";
import tmp from "tmp";
import { ZipFile } from "yazl";
import ZipHelper from "./ZipHelper";
async function writeZip(
entries: Record<string, string>,
postfix = ".zip"
): Promise<string> {
const zip = new JSZip();
const zip = new ZipFile();
for (const [name, content] of Object.entries(entries)) {
zip.file(name, content);
zip.addBuffer(Buffer.from(content), name);
}
const buffer = await zip.generateAsync({ type: "nodebuffer" });
const zipPath = tmp.fileSync({ postfix }).name;
await fs.writeFile(zipPath, buffer);
await new Promise<void>((resolve, reject) => {
const dest = fs
.createWriteStream(zipPath)
.on("finish", () => resolve())
.on("error", reject);
zip.outputStream.on("error", reject).pipe(dest);
zip.end();
});
return zipPath;
}
server/utils/ZipHelper.ts
+10 -53
View File
@@ -1,9 +1,9 @@
import path from "node:path";
import fs from "fs-extra";
import type JSZip from "jszip";
import tmp from "tmp";
import type { Entry } from "yauzl";
import yauzl, { validateFileName } from "yauzl";
import type { ZipFile } from "yazl";
import { bytesToHumanReadable } from "@shared/utils/files";
import { ValidationError } from "@server/errors";
import Logger from "@server/logging/Logger";
@@ -41,27 +41,16 @@ export interface ZipTreeNode {
@trace()
export default class ZipHelper {
public static defaultStreamOptions: JSZip.JSZipGeneratorOptions<"nodebuffer"> =
{
type: "nodebuffer",
streamFiles: true,
compression: "DEFLATE",
compressionOptions: {
level: 5,
},
};
/**
* Write a zip file to a temporary disk location
* Write a zip file to a temporary disk location.
*
* @deprecated Use `extract` instead
* @param zip JSZip object
* @returns pathname of the temporary file where the zip was written to disk
* The caller is responsible for adding entries to the `ZipFile`; this method
* calls `end()` and waits for the output stream to drain to disk.
*
* @param zip yazl ZipFile object with entries already added.
* @returns pathname of the temporary file where the zip was written to disk.
*/
public static async toTmpFile(
zip: JSZip,
options?: JSZip.JSZipGeneratorOptions<"nodebuffer">
): Promise<string> {
public static async toTmpFile(zip: ZipFile): Promise<string> {
Logger.debug("utils", "Creating tmp file…");
return new Promise((resolve, reject) => {
tmp.file(
@@ -74,11 +63,6 @@ export default class ZipHelper {
return reject(err);
}
let previousMetadata: JSZip.JSZipMetadata = {
percent: 0,
currentFile: null,
};
const handleError = (error: Error) => {
dest.destroy();
fs.remove(filePath)
@@ -98,35 +82,8 @@ export default class ZipHelper {
})
.on("error", handleError);
zip
.generateNodeStream(
{
...this.defaultStreamOptions,
...options,
},
(metadata) => {
if (metadata.currentFile !== previousMetadata.currentFile) {
const percent = Math.round(metadata.percent);
const memory = process.memoryUsage();
previousMetadata = {
currentFile: metadata.currentFile,
percent,
};
Logger.debug(
"utils",
`Writing zip file progress… ${percent}%`,
{
currentFile: metadata.currentFile,
memory: bytesToHumanReadable(memory.rss),
}
);
}
}
)
.on("error", handleError)
.pipe(dest)
.on("error", handleError);
zip.outputStream.on("error", handleError).pipe(dest);
zip.end();
}
);
});
server/utils/koa.ts
+35 -1
View File
@@ -1,6 +1,8 @@
import contentDisposition from "content-disposition";
import type formidable from "formidable";
import type { Request } from "koa";
import type { Context, Request } from "koa";
import { isArray } from "es-toolkit/compat";
import { ZipFile } from "yazl";
/**
* Get the first file from an incoming koa request
@@ -23,3 +25,35 @@ export const getFileFromRequest = (
return isArray(file) ? file[0] : file;
};
/**
* Stream a freshly-built zip archive as the response body. The supplied
* `build` callback receives a yazl ZipFile to populate with entries; the
* helper handles response headers, error forwarding, and finalizing the
* archive once `build` resolves.
*
* @param ctx The koa context to write the response to.
* @param fileName The filename to advertise in the Content-Disposition header.
* @param build Callback that adds entries to the provided ZipFile.
*/
export const streamZipResponse = async (
ctx: Context,
fileName: string,
build: (zip: ZipFile) => void | Promise<void>
): Promise<void> => {
const zip = new ZipFile();
await build(zip);
ctx.set("Content-Type", "application/zip");
ctx.set(
"Content-Disposition",
contentDisposition(fileName, { type: "attachment" })
);
zip.outputStream.on("error", (err) => {
ctx.app.emit("error", err, ctx);
ctx.res.destroy(err);
});
ctx.body = zip.outputStream;
zip.end();
};
yarn.lock
+28 -2
View File
@@ -8252,6 +8252,15 @@ __metadata:
languageName: node
linkType: hard
"@types/yazl@npm:^2.4.6":
version: 2.4.6
resolution: "@types/yazl@npm:2.4.6"
dependencies:
"@types/node": "npm:*"
checksum: 10c0/6145fd1025e592747ed1331d88edb6d57dfd0f514812420889ecc880e0728b22c4d4214d5062b36164c4c40ca271266968a1571118ba6521796c76ac0b18c99c
languageName: node
linkType: hard
"@upsetjs/venn.js@npm:^2.0.0":
version: 2.0.0
resolution: "@upsetjs/venn.js@npm:2.0.0"
@@ -9157,6 +9166,13 @@ __metadata:
languageName: node
linkType: hard
"buffer-crc32@npm:^1.0.0":
version: 1.0.0
resolution: "buffer-crc32@npm:1.0.0"
checksum: 10c0/8b86e161cee4bb48d5fa622cbae4c18f25e4857e5203b89e23de59e627ab26beb82d9d7999f2b8de02580165f61f83f997beaf02980cdf06affd175b651921ab
languageName: node
linkType: hard
"buffer-crc32@npm:~0.2.3":
version: 0.2.13
resolution: "buffer-crc32@npm:0.2.13"
@@ -13846,7 +13862,7 @@ __metadata:
languageName: node
linkType: hard
"jszip@npm:^3.10.1, jszip@npm:^3.7.1":
"jszip@npm:^3.7.1":
version: 3.10.1
resolution: "jszip@npm:3.10.1"
dependencies:
@@ -15858,6 +15874,7 @@ __metadata:
"@types/utf8": "npm:^3.0.3"
"@types/validator": "npm:^13.15.10"
"@types/yauzl": "npm:^2.10.3"
"@types/yazl": "npm:^2.4.6"
"@vitejs/plugin-react-oxc": "npm:^0.2.3"
"@vitest/ui": "npm:^4.1.6"
addressparser: "npm:^1.0.1"
@@ -15912,7 +15929,6 @@ __metadata:
js-yaml: "npm:^4.1.1"
jsdom: "npm:^22.1.0"
jsonwebtoken: "npm:^9.0.3"
jszip: "npm:^3.10.1"
katex: "npm:^0.16.45"
kbar: "npm:0.1.0-beta.48"
koa: "npm:^3.0.3"
@@ -16047,6 +16063,7 @@ __metadata:
y-prosemirror: "npm:^1.3.7"
y-protocols: "npm:^1.0.7"
yauzl: "npm:^3.2.1"
yazl: "npm:^3.3.1"
yjs: "npm:^13.6.30"
zod: "npm:^4.3.6"
languageName: unknown
@@ -21547,6 +21564,15 @@ __metadata:
languageName: node
linkType: hard
"yazl@npm:^3.3.1":
version: 3.3.1
resolution: "yazl@npm:3.3.1"
dependencies:
buffer-crc32: "npm:^1.0.0"
checksum: 10c0/8290dffd8afc4ea32459c0ce5d7e22af250515aa0170f289cbc896530d3cd40c5be4788b5e3e7aa5cd21889778071f424e1824faadbd257dbd5ff80fb60d1fa6
languageName: node
linkType: hard
"yjs@npm:^13.6.30":
version: 13.6.30
resolution: "yjs@npm:13.6.30"