GitHub/outline
1
0
Fork 0
mirror of https://github.com/outline/outline.git synced 2026-06-13 11:25:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

chore: Migrate from JSZip to Yazl (#12408)

Browse Source 
* chore: Migrate from JSZip to Yazl

* Add koa stream helper, PR feedback
This commit is contained in:
Tom Moor
2026-05-21 23:27:23 -04:00
committed by GitHub
parent d43280f08e
commit 8d44a0fd92
13 changed files with 213 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/queues/tasks/ExportDocumentTreeTask.ts
+34 -40
View File
@@ -1,6 +1,6 @@
import path from "node:path";
import type JSZip from "jszip";
import { escapeRegExp } from "es-toolkit/compat";
import type { ZipFile } from "yazl";
import type { NavigationNode } from "@shared/types";
import { FileOperationFormat } from "@shared/types";
import Logger from "@server/logging/Logger";
@@ -17,7 +17,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
/**
* Exports the document tree to the given zip instance.
*
* @param zip The JSZip instance to add files to
* @param zip The yazl ZipFile to add files to
* @param documentId The document ID to export
* @param pathInZip The path in the zip to add the document to
* @param format The format to export in
@@ -30,7 +30,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
includeAttachments,
pathMap,
}: {
zip: JSZip;
zip: ZipFile;
pathInZip: string;
documentId: string;
format: FileOperationFormat;
@@ -64,38 +64,33 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
// Add any referenced attachments to the zip file and replace the
// reference in the document with the path to the attachment in the zip
await Promise.all(
attachments.map(async (attachment) => {
Logger.debug("task", `Adding attachment to archive`, {
documentId,
key: attachment.key,
for (const attachment of attachments) {
Logger.debug("task", `Adding attachment to archive`, {
documentId,
key: attachment.key,
});
const dir = path.dirname(pathInZip);
let buffer: Buffer;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
buffer = Buffer.from("");
}
zip.addBuffer(buffer, path.join(dir, attachment.key), {
mtime: attachment.updatedAt,
});
const dir = path.dirname(pathInZip);
zip.file(
path.join(dir, attachment.key),
new Promise<Buffer>((resolve) => {
attachment.buffer.then(resolve).catch((err) => {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
resolve(Buffer.from(""));
});
}),
{
date: attachment.updatedAt,
createFolders: true,
}
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
encodeURI(attachment.key)
);
})
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
encodeURI(attachment.key)
);
}
// Replace any internal links with relative paths to the document in the zip
const internalLinks = [
@@ -117,10 +112,9 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
});
// Finally, add the document to the zip file
zip.file(pathInZip, text, {
date: document.updatedAt,
createFolders: true,
comment: JSON.stringify({
zip.addBuffer(Buffer.from(text), pathInZip, {
mtime: document.updatedAt,
fileComment: JSON.stringify({
createdAt: document.createdAt,
updatedAt: document.updatedAt,
}),
@@ -131,7 +125,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
* Exports the documents and attachments in the given collections to a zip file
* and returns the path to the zip file in tmp.
*
* @param zip The JSZip instance to add files to
* @param zip The yazl ZipFile to add files to
* @param collections The collections to export
* @param format The format to export in
* @param includeAttachments Whether to include attachments in the export
@@ -139,7 +133,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
* @returns The path to the zip file in tmp.
*/
protected async addCollectionsToArchive(
zip: JSZip,
zip: ZipFile,
collections: Collection[],
format: FileOperationFormat,
includeAttachments = true
@@ -178,7 +172,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
document: Document;
format: FileOperationFormat;
documentStructure: NavigationNode[];
zip: JSZip;
zip: ZipFile;
}) {
const pathMap = new Map<string, string>();
server/queues/tasks/ExportHTMLZipTask.ts
+3 -3
View File
@@ -1,4 +1,4 @@
import JSZip from "jszip";
import { ZipFile } from "yazl";
import type { NavigationNode } from "@shared/types";
import { FileOperationFormat } from "@shared/types";
import type { Collection, FileOperation } from "@server/models";
@@ -10,7 +10,7 @@ export default class ExportHTMLZipTask extends ExportDocumentTreeTask {
collections: Collection[],
fileOperation: FileOperation
) {
const zip = new JSZip();
const zip = new ZipFile();
return await this.addCollectionsToArchive(
zip,
@@ -24,7 +24,7 @@ export default class ExportHTMLZipTask extends ExportDocumentTreeTask {
document: Document,
documentStructure: NavigationNode[]
): Promise<string> {
const zip = new JSZip();
const zip = new ZipFile();
return await this.addDocumentToArchive({
document,
server/queues/tasks/ExportJSONTask.ts
+41 -39
View File
@@ -1,4 +1,4 @@
import JSZip from "jszip";
import { ZipFile } from "yazl";
import { omit } from "es-toolkit/compat";
import type { NavigationNode } from "@shared/types";
import env from "@server/env";
@@ -19,7 +19,7 @@ export default class ExportJSONTask extends ExportTask {
collections: Collection[],
fileOperation: FileOperation
) {
const zip = new JSZip();
const zip = new ZipFile();
const usedFilenames = new Set<string>();
// serial to avoid overloading, slow and steady wins the race
@@ -44,7 +44,10 @@ export default class ExportJSONTask extends ExportTask {
return ZipHelper.toTmpFile(zip);
}
private async addMetadataToArchive(zip: JSZip, fileOperation: FileOperation) {
private async addMetadataToArchive(
zip: ZipFile,
fileOperation: FileOperation
) {
const user = await fileOperation.$get("user");
const metadata: JSONExportMetadata = {
@@ -55,16 +58,18 @@ export default class ExportJSONTask extends ExportTask {
createdByEmail: user?.email ?? null,
};
zip.file(
`metadata.json`,
env.isDevelopment
? JSON.stringify(metadata, null, 2)
: JSON.stringify(metadata)
zip.addBuffer(
Buffer.from(
env.isDevelopment
? JSON.stringify(metadata, null, 2)
: JSON.stringify(metadata)
),
`metadata.json`
);
}
private async addCollectionToArchive(
zip: JSZip,
zip: ZipFile,
collection: Collection,
includeAttachments: boolean,
filename: string
@@ -82,32 +87,27 @@ export default class ExportJSONTask extends ExportTask {
};
async function addAttachments(attachments: Attachment[]) {
await Promise.all(
attachments.map(async (attachment) => {
zip.file(
attachment.key,
new Promise<Buffer>((resolve) => {
attachment.buffer.then(resolve).catch((err) => {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
resolve(Buffer.from(""));
});
}),
{
date: attachment.updatedAt,
createFolders: true,
}
);
for (const attachment of attachments) {
let buffer: Buffer;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
buffer = Buffer.from("");
}
zip.addBuffer(buffer, attachment.key, {
mtime: attachment.updatedAt,
});
output.attachments[attachment.id] = {
...omit(presentAttachment(attachment), "url"),
key: attachment.key,
};
})
);
output.attachments[attachment.id] = {
...omit(presentAttachment(attachment), "url"),
key: attachment.key,
};
}
}
async function addDocumentTree(nodes: NavigationNode[]) {
@@ -175,11 +175,13 @@ export default class ExportJSONTask extends ExportTask {
await addDocumentTree(collection.documentStructure);
}
zip.file(
`${filename}.json`,
env.isDevelopment
? JSON.stringify(output, null, 2)
: JSON.stringify(output)
zip.addBuffer(
Buffer.from(
env.isDevelopment
? JSON.stringify(output, null, 2)
: JSON.stringify(output)
),
`${filename}.json`
);
}
server/queues/tasks/ExportMarkdownZipTask.ts
+3 -3
View File
@@ -1,4 +1,4 @@
import JSZip from "jszip";
import { ZipFile } from "yazl";
import type { NavigationNode } from "@shared/types";
import { FileOperationFormat } from "@shared/types";
import type { Collection, FileOperation } from "@server/models";
@@ -10,7 +10,7 @@ export default class ExportMarkdownZipTask extends ExportDocumentTreeTask {
collections: Collection[],
fileOperation: FileOperation
) {
const zip = new JSZip();
const zip = new ZipFile();
return await this.addCollectionsToArchive(
zip,
@@ -24,7 +24,7 @@ export default class ExportMarkdownZipTask extends ExportDocumentTreeTask {
document: Document,
documentStructure: NavigationNode[]
): Promise<string> {
const zip = new JSZip();
const zip = new ZipFile();
return await this.addDocumentToArchive({
document,
server/routes/api/documents/documents.ts
+19 -36
View File
@@ -3,7 +3,6 @@ import fractionalIndex from "fractional-index";
import fs from "fs-extra";
import invariant from "invariant";
import contentDisposition from "content-disposition";
import JSZip from "jszip";
import Router from "koa-router";
import { escapeRegExp, has, remove, uniq } from "es-toolkit/compat";
import mime from "mime-types";
@@ -84,8 +83,8 @@ import EmptyTrashTask from "@server/queues/tasks/EmptyTrashTask";
import FileStorage from "@server/storage/files";
import type { APIContext } from "@server/types";
import { RateLimiterStrategy } from "@server/utils/RateLimiter";
import ZipHelper from "@server/utils/ZipHelper";
import { convertBareUrlsToEmbedMarkdown } from "@server/utils/embeds";
import { streamZipResponse } from "@server/utils/koa";
import { getTeamFromContext } from "@server/utils/passport";
import { assertPresent } from "@server/validation";
import pagination, { paginateQuery } from "../middlewares/pagination";
@@ -890,51 +889,35 @@ router.post(
return;
}
const zip = new JSZip();
await Promise.all(
attachments.map(async (attachment) => {
await streamZipResponse(ctx, `${fileName}.zip`, async (zip) => {
for (const attachment of attachments) {
const location = path.join(
"attachments",
`${attachment.id}.${mime.extension(attachment.contentType)}`
);
zip.file(
location,
new Promise<Buffer>((resolve) => {
attachment.buffer.then(resolve).catch((err) => {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
resolve(Buffer.from(""));
});
}),
{
date: attachment.updatedAt,
createFolders: true,
}
);
let buffer: Buffer;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
buffer = Buffer.from("");
}
zip.addBuffer(buffer, location, { mtime: attachment.updatedAt });
content = content.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
location
);
})
);
}
zip.file(`${fileName}.${extension}`, content, {
date: document.updatedAt,
zip.addBuffer(Buffer.from(content), `${fileName}.${extension}`, {
mtime: document.updatedAt,
});
});
ctx.set("Content-Type", "application/zip");
ctx.set(
"Content-Disposition",
contentDisposition(`${fileName}.zip`, {
type: "attachment",
})
);
ctx.body = zip.generateNodeStream(ZipHelper.defaultStreamOptions);
}
);
server/routes/api/middlewares/apiResponse.ts
+1 -4
View File
@@ -11,10 +11,7 @@ export default function apiResponse() {
typeof ctx.body === "object" &&
!(ctx.body instanceof Readable) &&
!(ctx.body instanceof stream.Readable) &&
!(ctx.body instanceof Buffer) &&
// JSZip returns a wrapped stream instance that is not a true readable stream
// and not exported from the module either, so we must identify it like so.
!(ctx.body && "_readableState" in ctx.body)
!(ctx.body instanceof Buffer)
) {
ctx.body = {
...ctx.body,
server/routes/api/revisions/revisions.ts
+19 -36
View File
@@ -1,7 +1,6 @@
import path from "node:path";
import Router from "koa-router";
import contentDisposition from "content-disposition";
import JSZip from "jszip";
import { escapeRegExp } from "es-toolkit/compat";
import mime from "mime-types";
import { UserRole } from "@shared/types";
@@ -20,7 +19,7 @@ import { authorize } from "@server/policies";
import { presentPolicies, presentRevision } from "@server/presenters";
import type { APIContext } from "@server/types";
import { RateLimiterStrategy } from "@server/utils/RateLimiter";
import ZipHelper from "@server/utils/ZipHelper";
import { streamZipResponse } from "@server/utils/koa";
import pagination from "../middlewares/pagination";
import * as T from "./schema";
@@ -195,51 +194,35 @@ router.post(
return;
}
const zip = new JSZip();
await Promise.all(
attachments.map(async (attachment) => {
await streamZipResponse(ctx, `${fileName}.zip`, async (zip) => {
for (const attachment of attachments) {
const location = path.join(
"attachments",
`${attachment.id}.${mime.extension(attachment.contentType)}`
);
zip.file(
location,
new Promise<Buffer>((resolve) => {
attachment.buffer.then(resolve).catch((err) => {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
resolve(Buffer.from(""));
});
}),
{
date: attachment.updatedAt,
createFolders: true,
}
);
let buffer: Buffer;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: err.message,
});
buffer = Buffer.from("");
}
zip.addBuffer(buffer, location, { mtime: attachment.updatedAt });
content = content.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
location
);
})
);
}
zip.file(`${fileName}.${extension}`, content, {
date: revision.updatedAt,
zip.addBuffer(Buffer.from(content), `${fileName}.${extension}`, {
mtime: revision.updatedAt,
});
});
ctx.set("Content-Type", "application/zip");
ctx.set(
"Content-Disposition",
contentDisposition(`${fileName}.zip`, {
type: "attachment",
})
);
ctx.body = zip.generateNodeStream(ZipHelper.defaultStreamOptions);
}
);
server/typings/yazl.d.ts
Vendored
+7
View File
@@ -0,0 +1,7 @@
import "yazl";
declare module "yazl" {
interface Options {
fileComment: string | Buffer;
}
}
server/utils/ZipHelper.test.ts
+11 -5
View File
@@ -1,20 +1,26 @@
import path from "node:path";
import fs from "fs-extra";
import JSZip from "jszip";
import tmp from "tmp";
import { ZipFile } from "yazl";
import ZipHelper from "./ZipHelper";
async function writeZip(
entries: Record<string, string>,
postfix = ".zip"
): Promise<string> {
const zip = new JSZip();
const zip = new ZipFile();
for (const [name, content] of Object.entries(entries)) {
zip.file(name, content);
zip.addBuffer(Buffer.from(content), name);
}
const buffer = await zip.generateAsync({ type: "nodebuffer" });
const zipPath = tmp.fileSync({ postfix }).name;
await fs.writeFile(zipPath, buffer);
await new Promise<void>((resolve, reject) => {
const dest = fs
.createWriteStream(zipPath)
.on("finish", () => resolve())
.on("error", reject);
zip.outputStream.on("error", reject).pipe(dest);
zip.end();
});
return zipPath;
}
server/utils/ZipHelper.ts
+10 -53
View File
@@ -1,9 +1,9 @@
import path from "node:path";
import fs from "fs-extra";
import type JSZip from "jszip";
import tmp from "tmp";
import type { Entry } from "yauzl";
import yauzl, { validateFileName } from "yauzl";
import type { ZipFile } from "yazl";
import { bytesToHumanReadable } from "@shared/utils/files";
import { ValidationError } from "@server/errors";
import Logger from "@server/logging/Logger";
@@ -41,27 +41,16 @@ export interface ZipTreeNode {
@trace()
export default class ZipHelper {
public static defaultStreamOptions: JSZip.JSZipGeneratorOptions<"nodebuffer"> =
{
type: "nodebuffer",
streamFiles: true,
compression: "DEFLATE",
compressionOptions: {
level: 5,
},
};
/**
* Write a zip file to a temporary disk location
* Write a zip file to a temporary disk location.
*
* @deprecated Use `extract` instead
* @param zip JSZip object
* @returns pathname of the temporary file where the zip was written to disk
* The caller is responsible for adding entries to the `ZipFile`; this method
* calls `end()` and waits for the output stream to drain to disk.
*
* @param zip yazl ZipFile object with entries already added.
* @returns pathname of the temporary file where the zip was written to disk.
*/
public static async toTmpFile(
zip: JSZip,
options?: JSZip.JSZipGeneratorOptions<"nodebuffer">
): Promise<string> {
public static async toTmpFile(zip: ZipFile): Promise<string> {
Logger.debug("utils", "Creating tmp file…");
return new Promise((resolve, reject) => {
tmp.file(
@@ -74,11 +63,6 @@ export default class ZipHelper {
return reject(err);
}
let previousMetadata: JSZip.JSZipMetadata = {
percent: 0,
currentFile: null,
};
const handleError = (error: Error) => {
dest.destroy();
fs.remove(filePath)
@@ -98,35 +82,8 @@ export default class ZipHelper {
})
.on("error", handleError);
zip
.generateNodeStream(
{
...this.defaultStreamOptions,
...options,
},
(metadata) => {
if (metadata.currentFile !== previousMetadata.currentFile) {
const percent = Math.round(metadata.percent);
const memory = process.memoryUsage();
previousMetadata = {
currentFile: metadata.currentFile,
percent,
};
Logger.debug(
"utils",
`Writing zip file progress… ${percent}%`,
{
currentFile: metadata.currentFile,
memory: bytesToHumanReadable(memory.rss),
}
);
}
}
)
.on("error", handleError)
.pipe(dest)
.on("error", handleError);
zip.outputStream.on("error", handleError).pipe(dest);
zip.end();
}
);
});
server/utils/koa.ts
+35 -1
View File
@@ -1,6 +1,8 @@
import contentDisposition from "content-disposition";
import type formidable from "formidable";
import type { Request } from "koa";
import type { Context, Request } from "koa";
import { isArray } from "es-toolkit/compat";
import { ZipFile } from "yazl";
/**
* Get the first file from an incoming koa request
@@ -23,3 +25,35 @@ export const getFileFromRequest = (
return isArray(file) ? file[0] : file;
};
/**
* Stream a freshly-built zip archive as the response body. The supplied
* `build` callback receives a yazl ZipFile to populate with entries; the
* helper handles response headers, error forwarding, and finalizing the
* archive once `build` resolves.
*
* @param ctx The koa context to write the response to.
* @param fileName The filename to advertise in the Content-Disposition header.
* @param build Callback that adds entries to the provided ZipFile.
*/
export const streamZipResponse = async (
ctx: Context,
fileName: string,
build: (zip: ZipFile) => void | Promise<void>
): Promise<void> => {
const zip = new ZipFile();
await build(zip);
ctx.set("Content-Type", "application/zip");
ctx.set(
"Content-Disposition",
contentDisposition(fileName, { type: "attachment" })
);
zip.outputStream.on("error", (err) => {
ctx.app.emit("error", err, ctx);
ctx.res.destroy(err);
});
ctx.body = zip.outputStream;
zip.end();
};