mirror of
https://github.com/outline/outline.git
synced 2026-06-13 19:35:02 +03:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 26be6dcf98 | |||
| a3910ce6d1 | |||
| f9476770ce | |||
| 2e018e74b8 | |||
| a11ab56117 | |||
| 66e4ec32ed | |||
| bde9d5fbf4 | |||
| 70bb878a8c | |||
| 4237377d47 | |||
| a30f6b717b | |||
| 1edc23c5ae |
@@ -11,7 +11,7 @@ env:
|
||||
DATABASE_URL: postgres://postgres:password@localhost:5432/outline_test
|
||||
REDIS_URL: redis://127.0.0.1:6379
|
||||
URL: http://localhost:3000
|
||||
NODE_OPTIONS: --max-old-space-size=8192
|
||||
NODE_OPTIONS: --max-old-space-size=8000
|
||||
SECRET_KEY: F0E5AD933D7F6FD8F4DBB3E038C501C052DC0593C686D21ACB30AE205D2F634B
|
||||
UTILS_SECRET: 123456
|
||||
SLACK_VERIFICATION_TOKEN: 123456
|
||||
|
||||
+19
-179
@@ -3,32 +3,25 @@ name: Docker
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
- 'v*'
|
||||
|
||||
env:
|
||||
IMAGE_NAME: outlinewiki/outline
|
||||
BASE_IMAGE_NAME: outlinewiki/outline-base
|
||||
|
||||
jobs:
|
||||
build-arm:
|
||||
runs-on: ubicloud-standard-8-arm
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Docker base meta
|
||||
id: base_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.BASE_IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -36,177 +29,24 @@ jobs:
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build and push base image
|
||||
id: base_build
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.base
|
||||
labels: ${{ steps.base_meta.outputs.labels }}
|
||||
tags: ${{ env.BASE_IMAGE_NAME }}
|
||||
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
||||
platforms: linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
pull: false
|
||||
push: true
|
||||
tags: ${{ env.BASE_IMAGE_NAME }}:latest
|
||||
platforms: linux/amd64,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
- name: Extract version
|
||||
id: version
|
||||
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build and push
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
- name: Build and push main image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
tags: ${{ env.IMAGE_NAME }}
|
||||
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
||||
platforms: linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
pull: false
|
||||
build-args: |
|
||||
BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}@${{ steps.base_build.outputs.digest }}
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
mkdir -p ${{ runner.temp }}/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "${{ runner.temp }}/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-linux-arm64
|
||||
path: ${{ runner.temp }}/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
build-amd:
|
||||
runs-on: ubicloud-standard-8
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Docker base meta
|
||||
id: base_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.BASE_IMAGE_NAME }}
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build and push base image
|
||||
id: base_build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.base
|
||||
labels: ${{ steps.base_meta.outputs.labels }}
|
||||
tags: ${{ env.BASE_IMAGE_NAME }}
|
||||
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
||||
platforms: linux/amd64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
pull: false
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
|
||||
- name: Build and push
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
tags: ${{ env.IMAGE_NAME }}
|
||||
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
||||
platforms: linux/amd64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
pull: false
|
||||
build-args: |
|
||||
BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}@${{ steps.base_build.outputs.digest }}
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
mkdir -p ${{ runner.temp }}/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "${{ runner.temp }}/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-linux-amd64
|
||||
path: ${{ runner.temp }}/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs:
|
||||
- build-amd
|
||||
- build-arm
|
||||
steps:
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/digests
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
|
||||
- name: Create manifest list and push
|
||||
working-directory: ${{ runner.temp }}/digests
|
||||
run: |
|
||||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *)
|
||||
|
||||
- name: Inspect image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
|
||||
${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.IMAGE_NAME }}:${{ steps.version.outputs.VERSION }}
|
||||
|
||||
+2
-3
@@ -1,6 +1,5 @@
|
||||
ARG APP_PATH=/opt/outline
|
||||
ARG BASE_IMAGE=outlinewiki/outline-base
|
||||
FROM ${BASE_IMAGE} AS base
|
||||
FROM outlinewiki/outline-base AS base
|
||||
|
||||
ARG APP_PATH
|
||||
WORKDIR $APP_PATH
|
||||
@@ -31,7 +30,7 @@ RUN addgroup --gid 1001 nodejs && \
|
||||
adduser --uid 1001 --ingroup nodejs nodejs && \
|
||||
chown -R nodejs:nodejs $APP_PATH/build && \
|
||||
mkdir -p /var/lib/outline && \
|
||||
chown -R nodejs:nodejs /var/lib/outline
|
||||
chown -R nodejs:nodejs /var/lib/outline
|
||||
|
||||
ENV FILE_STORAGE_LOCAL_ROOT_DIR=/var/lib/outline/data
|
||||
RUN mkdir -p "$FILE_STORAGE_LOCAL_ROOT_DIR" && \
|
||||
|
||||
+1
-4
@@ -1,14 +1,11 @@
|
||||
ARG APP_PATH=/opt/outline
|
||||
FROM node:20 AS deps
|
||||
FROM node:20-slim AS deps
|
||||
|
||||
ARG APP_PATH
|
||||
WORKDIR $APP_PATH
|
||||
COPY ./package.json ./yarn.lock ./
|
||||
COPY ./patches ./patches
|
||||
|
||||
RUN apt-get update && apt-get install -y cmake
|
||||
ENV NODE_OPTIONS="--max-old-space-size=24000"
|
||||
|
||||
RUN yarn install --no-optional --frozen-lockfile --network-timeout 1000000 && \
|
||||
yarn cache clean
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ Business Source License 1.1
|
||||
Parameters
|
||||
|
||||
Licensor: General Outline, Inc.
|
||||
Licensed Work: Outline 0.82.1-31
|
||||
Licensed Work: Outline 0.82.0
|
||||
The Licensed Work is (c) 2025 General Outline, Inc.
|
||||
Additional Use Grant: You may make use of the Licensed Work, provided that
|
||||
you may not use the Licensed Work for a Document
|
||||
@@ -15,7 +15,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that
|
||||
Licensed Work by creating teams and documents
|
||||
controlled by such third parties.
|
||||
|
||||
Change Date: 2029-04-11
|
||||
Change Date: 2029-02-15
|
||||
|
||||
Change License: Apache License, Version 2.0
|
||||
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
import { AnimatePresence } from "framer-motion";
|
||||
import { observer } from "mobx-react";
|
||||
import * as React from "react";
|
||||
import { Switch, Route, useLocation, matchPath } from "react-router-dom";
|
||||
import {
|
||||
Switch,
|
||||
Route,
|
||||
useLocation,
|
||||
matchPath,
|
||||
Redirect,
|
||||
} from "react-router-dom";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import ErrorSuspended from "~/scenes/Errors/ErrorSuspended";
|
||||
import Layout from "~/components/Layout";
|
||||
@@ -10,6 +16,7 @@ import Sidebar from "~/components/Sidebar";
|
||||
import SidebarRight from "~/components/Sidebar/Right";
|
||||
import SettingsSidebar from "~/components/Sidebar/Settings";
|
||||
import useCurrentTeam from "~/hooks/useCurrentTeam";
|
||||
import { usePostLoginPath } from "~/hooks/useLastVisitedPath";
|
||||
import usePolicy from "~/hooks/usePolicy";
|
||||
import useStores from "~/hooks/useStores";
|
||||
import history from "~/utils/history";
|
||||
@@ -48,6 +55,7 @@ const AuthenticatedLayout: React.FC = ({ children }: Props) => {
|
||||
const can = usePolicy(ui.activeDocumentId);
|
||||
const canCollection = usePolicy(ui.activeCollectionId);
|
||||
const team = useCurrentTeam();
|
||||
const [spendPostLoginPath] = usePostLoginPath();
|
||||
|
||||
const goToSearch = (ev: KeyboardEvent) => {
|
||||
if (!ev.metaKey && !ev.ctrlKey) {
|
||||
@@ -72,6 +80,11 @@ const AuthenticatedLayout: React.FC = ({ children }: Props) => {
|
||||
return <ErrorSuspended />;
|
||||
}
|
||||
|
||||
const postLoginPath = spendPostLoginPath();
|
||||
if (postLoginPath) {
|
||||
return <Redirect to={postLoginPath} />;
|
||||
}
|
||||
|
||||
const sidebar = (
|
||||
<Fade>
|
||||
<Switch>
|
||||
|
||||
+1
-8
@@ -2,7 +2,7 @@ import { observer } from "mobx-react";
|
||||
import { HomeIcon } from "outline-icons";
|
||||
import * as React from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Switch, Route, Redirect } from "react-router-dom";
|
||||
import { Switch, Route } from "react-router-dom";
|
||||
import styled from "styled-components";
|
||||
import { s } from "@shared/styles";
|
||||
import { Action } from "~/components/Actions";
|
||||
@@ -18,7 +18,6 @@ import Tab from "~/components/Tab";
|
||||
import Tabs from "~/components/Tabs";
|
||||
import useCurrentTeam from "~/hooks/useCurrentTeam";
|
||||
import useCurrentUser from "~/hooks/useCurrentUser";
|
||||
import { usePostLoginPath } from "~/hooks/useLastVisitedPath";
|
||||
import { usePinnedDocuments } from "~/hooks/usePinnedDocuments";
|
||||
import usePolicy from "~/hooks/usePolicy";
|
||||
import useStores from "~/hooks/useStores";
|
||||
@@ -29,16 +28,10 @@ function Home() {
|
||||
const team = useCurrentTeam();
|
||||
const user = useCurrentUser();
|
||||
const { t } = useTranslation();
|
||||
const [spendPostLoginPath] = usePostLoginPath();
|
||||
const userId = user?.id;
|
||||
const { pins, count } = usePinnedDocuments("home");
|
||||
const can = usePolicy(team);
|
||||
|
||||
const postLoginPath = spendPostLoginPath();
|
||||
if (postLoginPath) {
|
||||
return <Redirect to={postLoginPath} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<Scene
|
||||
icon={<HomeIcon />}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
import * as React from "react";
|
||||
import { Trans, useTranslation } from "react-i18next";
|
||||
import { FileOperationFormat } from "@shared/types";
|
||||
import useStores from "~/hooks/useStores";
|
||||
import DropToImport from "./DropToImport";
|
||||
import HelpDisclosure from "./HelpDisclosure";
|
||||
|
||||
function ImportNotionDialog() {
|
||||
const { t } = useTranslation();
|
||||
const { dialogs } = useStores();
|
||||
|
||||
return (
|
||||
<>
|
||||
<HelpDisclosure title={<Trans>Where do I find the file?</Trans>}>
|
||||
<Trans
|
||||
defaults="In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability."
|
||||
components={{
|
||||
em: <strong />,
|
||||
}}
|
||||
/>
|
||||
</HelpDisclosure>
|
||||
<DropToImport
|
||||
onSubmit={dialogs.closeAllModals}
|
||||
format={FileOperationFormat.Notion}
|
||||
>
|
||||
<>
|
||||
{t(
|
||||
`Drag and drop the zip file from Notion's HTML export option, or click to upload`
|
||||
)}
|
||||
</>
|
||||
</DropToImport>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default ImportNotionDialog;
|
||||
+8
-8
@@ -48,11 +48,11 @@
|
||||
"> 0.25%, not dead"
|
||||
],
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.774.0",
|
||||
"@aws-sdk/lib-storage": "3.774.0",
|
||||
"@aws-sdk/s3-presigned-post": "3.774.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.774.0",
|
||||
"@aws-sdk/signature-v4-crt": "^3.774.0",
|
||||
"@aws-sdk/client-s3": "3.777.0",
|
||||
"@aws-sdk/lib-storage": "3.777.0",
|
||||
"@aws-sdk/s3-presigned-post": "3.777.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.777.0",
|
||||
"@aws-sdk/signature-v4-crt": "^3.775.0",
|
||||
"@babel/core": "^7.26.10",
|
||||
"@babel/plugin-proposal-decorators": "^7.25.9",
|
||||
"@babel/plugin-transform-class-properties": "^7.25.9",
|
||||
@@ -248,7 +248,7 @@
|
||||
"uuid": "^8.3.2",
|
||||
"validator": "13.12.0",
|
||||
"vaul": "^1.1.2",
|
||||
"vite": "^5.4.15",
|
||||
"vite": "^5.4.16",
|
||||
"vite-plugin-pwa": "^0.20.3",
|
||||
"winston": "^3.17.0",
|
||||
"ws": "^7.5.10",
|
||||
@@ -377,5 +377,5 @@
|
||||
"rollup": "^4.5.1",
|
||||
"prismjs": "1.30.0"
|
||||
},
|
||||
"version": "0.82.1-31"
|
||||
}
|
||||
"version": "0.82.0"
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import { APIResponseError, APIErrorCode } from "@notionhq/client";
|
||||
import { ImportTaskInput, ImportTaskOutput } from "@shared/schema";
|
||||
import { IntegrationService, ProsemirrorDoc } from "@shared/types";
|
||||
import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { Integration } from "@server/models";
|
||||
import ImportTask from "@server/models/ImportTask";
|
||||
import APIImportTask, {
|
||||
@@ -39,7 +41,10 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
importTask.input.map(async (item) => this.processPage({ item, client }))
|
||||
);
|
||||
|
||||
const taskOutput: ImportTaskOutput = parsedPages.map((parsedPage) => ({
|
||||
// Filter out any null results (from pages/databases that couldn't be accessed)
|
||||
const validParsedPages = parsedPages.filter(Boolean) as ParsePageOutput[];
|
||||
|
||||
const taskOutput: ImportTaskOutput = validParsedPages.map((parsedPage) => ({
|
||||
externalId: parsedPage.externalId,
|
||||
title: parsedPage.title,
|
||||
emoji: parsedPage.emoji,
|
||||
@@ -50,7 +55,7 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
}));
|
||||
|
||||
const childTasksInput: ImportTaskInput<IntegrationService.Notion> =
|
||||
parsedPages.flatMap((parsedPage) =>
|
||||
validParsedPages.flatMap((parsedPage) =>
|
||||
parsedPage.children.map((childPage) => ({
|
||||
type: childPage.type,
|
||||
externalId: childPage.externalId,
|
||||
@@ -88,36 +93,55 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
}: {
|
||||
item: ImportTaskInput<IntegrationService.Notion>[number];
|
||||
client: NotionClient;
|
||||
}): Promise<ParsePageOutput> {
|
||||
}): Promise<ParsePageOutput | null> {
|
||||
const collectionExternalId = item.collectionExternalId ?? item.externalId;
|
||||
|
||||
// Convert Notion database to an empty page with "pages in database" as its children.
|
||||
if (item.type === PageType.Database) {
|
||||
const { pages, ...databaseInfo } = await client.fetchDatabase(
|
||||
item.externalId
|
||||
);
|
||||
try {
|
||||
// Convert Notion database to an empty page with "pages in database" as its children.
|
||||
if (item.type === PageType.Database) {
|
||||
const { pages, ...databaseInfo } = await client.fetchDatabase(
|
||||
item.externalId
|
||||
);
|
||||
|
||||
return {
|
||||
...databaseInfo,
|
||||
externalId: item.externalId,
|
||||
content: ProsemirrorHelper.getEmptyDocument() as ProsemirrorDoc,
|
||||
collectionExternalId,
|
||||
children: pages.map((page) => ({
|
||||
type: page.type,
|
||||
externalId: page.id,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const { blocks, ...pageInfo } = await client.fetchPage(item.externalId);
|
||||
|
||||
return {
|
||||
...databaseInfo,
|
||||
...pageInfo,
|
||||
externalId: item.externalId,
|
||||
content: ProsemirrorHelper.getEmptyDocument() as ProsemirrorDoc,
|
||||
content: NotionConverter.page({ children: blocks } as NotionPage),
|
||||
collectionExternalId,
|
||||
children: pages.map((page) => ({
|
||||
type: page.type,
|
||||
externalId: page.id,
|
||||
})),
|
||||
children: this.parseChildPages(blocks),
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof APIResponseError) {
|
||||
// Skip this page/database if it's not found or not accessible
|
||||
if (
|
||||
error.code === APIErrorCode.ObjectNotFound ||
|
||||
error.code === APIErrorCode.Unauthorized
|
||||
) {
|
||||
Logger.warn(
|
||||
`Skipping Notion ${
|
||||
item.type === PageType.Database ? "database" : "page"
|
||||
} ${item.externalId} - Error code: ${error.code} - ${error.message}`
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
// Re-throw other errors to be handled by the parent try/catch
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { blocks, ...pageInfo } = await client.fetchPage(item.externalId);
|
||||
|
||||
return {
|
||||
...pageInfo,
|
||||
externalId: item.externalId,
|
||||
content: NotionConverter.page({ children: blocks } as NotionPage),
|
||||
collectionExternalId,
|
||||
children: this.parseChildPages(blocks),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { Server } from "@hocuspocus/server";
|
||||
import WebSocket from "ws";
|
||||
import EDITOR_VERSION from "@shared/editor/version";
|
||||
import { sleep } from "@server/utils/timers";
|
||||
import { ConnectionLimitExtension } from "./ConnectionLimitExtension";
|
||||
import { EditorVersionExtension } from "./EditorVersionExtension";
|
||||
|
||||
jest.mock("@server/env", () => ({
|
||||
COLLABORATION_MAX_CLIENTS_PER_DOCUMENT: 2,
|
||||
}));
|
||||
|
||||
describe("ConnectionLimitExtension", () => {
|
||||
let server: typeof Server;
|
||||
let extension: ConnectionLimitExtension;
|
||||
const port = 12345;
|
||||
const url = `ws://localhost:${port}`;
|
||||
const documentName = "test";
|
||||
|
||||
beforeEach(async () => {
|
||||
extension = new ConnectionLimitExtension();
|
||||
server = Server.configure({
|
||||
port,
|
||||
extensions: [extension, new EditorVersionExtension()],
|
||||
});
|
||||
await server.listen();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await server.destroy();
|
||||
});
|
||||
|
||||
const getConnections = () =>
|
||||
extension.connectionsByDocument.get(documentName)?.size ?? 0;
|
||||
|
||||
const createWebSocket = (editorVersion = EDITOR_VERSION) =>
|
||||
new Promise<WebSocket>((resolve, reject) => {
|
||||
const ws = new WebSocket(
|
||||
`${url}/${documentName}?editorVersion=${editorVersion}`
|
||||
);
|
||||
ws.on("open", () => resolve(ws));
|
||||
ws.on("error", reject);
|
||||
});
|
||||
|
||||
it("should allow connections within limit", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
expect(ws1.readyState).toBe(WebSocket.OPEN);
|
||||
expect(ws2.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should close connections exceeding limit", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
const ws3 = await createWebSocket();
|
||||
await sleep(250);
|
||||
|
||||
expect(ws3.readyState).toBe(WebSocket.CLOSED);
|
||||
expect(ws2.readyState).toBe(WebSocket.OPEN);
|
||||
expect(ws1.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should handle connections closed by other extensions", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
|
||||
// Create a connection that will be closed by the EditorVersionExtension
|
||||
const ws2 = await createWebSocket("1.0.0");
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should allow new connection after disconnect", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
ws1.close();
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(1);
|
||||
|
||||
const ws3 = await createWebSocket();
|
||||
expect(ws3.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws2.close();
|
||||
ws3.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,10 @@
|
||||
import {
|
||||
Extension,
|
||||
connectedPayload,
|
||||
onConnectPayload,
|
||||
onDisconnectPayload,
|
||||
} from "@hocuspocus/server";
|
||||
import pluralize from "pluralize";
|
||||
import { TooManyConnections } from "@shared/collaboration/CloseEvents";
|
||||
import env from "@server/env";
|
||||
import Logger from "@server/logging/Logger";
|
||||
@@ -14,7 +16,7 @@ export class ConnectionLimitExtension implements Extension {
|
||||
/**
|
||||
* Map of documentId -> connection count
|
||||
*/
|
||||
connectionsByDocument: Map<string, Set<string>> = new Map();
|
||||
public connectionsByDocument: Map<string, Set<string>> = new Map();
|
||||
|
||||
/**
|
||||
* On disconnect hook
|
||||
@@ -34,23 +36,30 @@ export class ConnectionLimitExtension implements Extension {
|
||||
}
|
||||
}
|
||||
|
||||
const connectionCount = connections?.size ?? 0;
|
||||
Logger.debug(
|
||||
"multiplayer",
|
||||
`${connections?.size} connections to "${documentName}"`
|
||||
`${connectionCount} ${pluralize(
|
||||
"connection",
|
||||
connectionCount
|
||||
)} to "${documentName}"`
|
||||
);
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
/**
|
||||
* On connect hook
|
||||
* onConnect hook is called when a new connection has been established.
|
||||
* This is where we can check if the document has reached the maximum number of
|
||||
* connections and reject the connection if it has.
|
||||
*
|
||||
* @param data The connect payload
|
||||
* @returns Promise, resolving will allow the connection, rejecting will drop it
|
||||
* @param data The onConnect payload
|
||||
* @returns Promise, resolving will allow the connection, rejecting will drop.
|
||||
*/
|
||||
onConnect({ documentName, socketId }: withContext<onConnectPayload>) {
|
||||
onConnect({ documentName }: withContext<onConnectPayload>) {
|
||||
const connections =
|
||||
this.connectionsByDocument.get(documentName) || new Set();
|
||||
|
||||
if (connections?.size >= env.COLLABORATION_MAX_CLIENTS_PER_DOCUMENT) {
|
||||
Logger.info(
|
||||
"multiplayer",
|
||||
@@ -61,12 +70,30 @@ export class ConnectionLimitExtension implements Extension {
|
||||
return Promise.reject(TooManyConnections);
|
||||
}
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
/**
|
||||
* Connected hook is called after a new connection has been established.
|
||||
* We can safely update the connection count for the document.
|
||||
*
|
||||
* @param data The onConnect payload
|
||||
* @returns Promise
|
||||
*/
|
||||
connected({ documentName, socketId }: withContext<connectedPayload>) {
|
||||
const connections =
|
||||
this.connectionsByDocument.get(documentName) || new Set();
|
||||
|
||||
connections.add(socketId);
|
||||
this.connectionsByDocument.set(documentName, connections);
|
||||
const connectionCount = connections.size ?? 0;
|
||||
|
||||
Logger.debug(
|
||||
"multiplayer",
|
||||
`${connections.size} connections to "${documentName}"`
|
||||
`${connectionCount} ${pluralize(
|
||||
"connection",
|
||||
connectionCount
|
||||
)} to "${documentName}"`
|
||||
);
|
||||
|
||||
return Promise.resolve();
|
||||
|
||||
@@ -3,7 +3,6 @@ import slugify from "slugify";
|
||||
import { RESERVED_SUBDOMAINS } from "@shared/utils/domains";
|
||||
import { traceFunction } from "@server/logging/tracing";
|
||||
import { Team, Event } from "@server/models";
|
||||
import { generateAvatarUrl } from "@server/utils/avatars";
|
||||
|
||||
type Props = {
|
||||
/** The displayed name of the team */
|
||||
@@ -36,13 +35,10 @@ async function teamCreator({
|
||||
ip,
|
||||
transaction,
|
||||
}: Props): Promise<Team> {
|
||||
// If the service did not provide a logo/avatar then we attempt to generate
|
||||
// one via ClearBit, or fallback to colored initials in worst case scenario
|
||||
// If the service did not provide a logo/avatar then we'll use the default
|
||||
// avatar generation mechanism (colored initials)
|
||||
if (!avatarUrl || !avatarUrl.startsWith("http")) {
|
||||
avatarUrl = await generateAvatarUrl({
|
||||
domain,
|
||||
id: subdomain,
|
||||
});
|
||||
avatarUrl = null;
|
||||
}
|
||||
|
||||
const team = await Team.create(
|
||||
|
||||
@@ -6,7 +6,6 @@ import ExportJSONTask from "../tasks/ExportJSONTask";
|
||||
import ExportMarkdownZipTask from "../tasks/ExportMarkdownZipTask";
|
||||
import ImportJSONTask from "../tasks/ImportJSONTask";
|
||||
import ImportMarkdownZipTask from "../tasks/ImportMarkdownZipTask";
|
||||
import ImportNotionTask from "../tasks/ImportNotionTask";
|
||||
import BaseProcessor from "./BaseProcessor";
|
||||
|
||||
export default class FileOperationCreatedProcessor extends BaseProcessor {
|
||||
@@ -25,11 +24,6 @@ export default class FileOperationCreatedProcessor extends BaseProcessor {
|
||||
fileOperationId: event.modelId,
|
||||
});
|
||||
break;
|
||||
case FileOperationFormat.Notion:
|
||||
await ImportNotionTask.schedule({
|
||||
fileOperationId: event.modelId,
|
||||
});
|
||||
break;
|
||||
case FileOperationFormat.JSON:
|
||||
await ImportJSONTask.schedule({
|
||||
fileOperationId: event.modelId,
|
||||
|
||||
@@ -3,7 +3,12 @@ import chunk from "lodash/chunk";
|
||||
import keyBy from "lodash/keyBy";
|
||||
import truncate from "lodash/truncate";
|
||||
import { Fragment, Node } from "prosemirror-model";
|
||||
import { CreateOptions, CreationAttributes, Transaction } from "sequelize";
|
||||
import {
|
||||
CreateOptions,
|
||||
CreationAttributes,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from "sequelize";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomElement } from "@shared/random";
|
||||
import { ImportInput, ImportTaskInput } from "@shared/schema";
|
||||
@@ -154,45 +159,59 @@ export default abstract class ImportsProcessor<
|
||||
* @returns Promise that resolves when mapping and persistence is completed.
|
||||
*/
|
||||
private async onProcessed(importModel: Import<T>, transaction: Transaction) {
|
||||
const { collections } = await this.createCollectionsAndDocuments({
|
||||
importModel,
|
||||
transaction,
|
||||
});
|
||||
|
||||
// Once all collections and documents are created, update collection's document structure.
|
||||
// This ensures the root documents have the whole subtree available in the structure.
|
||||
for (const collection of collections) {
|
||||
await Document.unscoped().findAllInBatches<Document>(
|
||||
{
|
||||
where: { parentDocumentId: null, collectionId: collection.id },
|
||||
order: [
|
||||
["createdAt", "DESC"],
|
||||
["id", "ASC"],
|
||||
],
|
||||
transaction,
|
||||
},
|
||||
async (documents) => {
|
||||
for (const document of documents) {
|
||||
await collection.addDocumentToStructure(document, 0, {
|
||||
save: false,
|
||||
silent: true,
|
||||
transaction,
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
await collection.save({ silent: true, transaction });
|
||||
}
|
||||
|
||||
importModel.state = ImportState.Completed;
|
||||
importModel.error = null; // unset any error from previous attempts.
|
||||
await importModel.saveWithCtx(
|
||||
createContext({
|
||||
user: importModel.createdBy,
|
||||
try {
|
||||
const { collections } = await this.createCollectionsAndDocuments({
|
||||
importModel,
|
||||
transaction,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
// Once all collections and documents are created, update collection's document structure.
|
||||
// This ensures the root documents have the whole subtree available in the structure.
|
||||
for (const collection of collections) {
|
||||
await Document.unscoped().findAllInBatches<Document>(
|
||||
{
|
||||
where: { parentDocumentId: null, collectionId: collection.id },
|
||||
order: [
|
||||
["createdAt", "DESC"],
|
||||
["id", "ASC"],
|
||||
],
|
||||
transaction,
|
||||
},
|
||||
async (documents) => {
|
||||
for (const document of documents) {
|
||||
await collection.addDocumentToStructure(document, 0, {
|
||||
save: false,
|
||||
silent: true,
|
||||
transaction,
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
await collection.save({ silent: true, transaction });
|
||||
}
|
||||
|
||||
importModel.state = ImportState.Completed;
|
||||
importModel.error = null; // unset any error from previous attempts.
|
||||
await importModel.saveWithCtx(
|
||||
createContext({
|
||||
user: importModel.createdBy,
|
||||
transaction,
|
||||
})
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof UniqueConstraintError) {
|
||||
Logger.error(
|
||||
"ImportsProcessor persistence failed due to unique constraint error",
|
||||
err,
|
||||
{
|
||||
fields: err.fields,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -304,6 +323,15 @@ export default abstract class ImportsProcessor<
|
||||
|
||||
const output = outputMap[externalId];
|
||||
|
||||
// Skip this item if it has no output (likely due to an error during processing)
|
||||
if (!output) {
|
||||
Logger.debug(
|
||||
"processor",
|
||||
`Skipping item with no output: ${externalId}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const collectionItem = importInput[externalId];
|
||||
|
||||
const attachments = await Attachment.findAll({
|
||||
@@ -444,7 +472,7 @@ export default abstract class ImportsProcessor<
|
||||
importInput: Record<string, ImportInput<any>[number]>;
|
||||
actorId: string;
|
||||
}): ProsemirrorDoc {
|
||||
// special case when the doc content is empty
|
||||
// special case when the doc content is empty.
|
||||
if (!content.content.length) {
|
||||
return content;
|
||||
}
|
||||
|
||||
@@ -170,7 +170,7 @@ export default abstract class APIImportTask<
|
||||
await importTask.save({ transaction });
|
||||
|
||||
const associatedImport = importTask.import;
|
||||
associatedImport.documentCount += importTask.input.length;
|
||||
associatedImport.documentCount += taskOutputWithReplacements.length;
|
||||
await associatedImport.saveWithCtx(
|
||||
createContext({
|
||||
user: associatedImport.createdBy,
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
/* eslint-disable @typescript-eslint/no-empty-function */
|
||||
import path from "path";
|
||||
import { FileOperation } from "@server/models";
|
||||
import { buildFileOperation } from "@server/test/factories";
|
||||
import ImportNotionTask from "./ImportNotionTask";
|
||||
|
||||
describe("ImportNotionTask", () => {
|
||||
it("should import successfully from a Markdown export", async () => {
|
||||
const fileOperation = await buildFileOperation();
|
||||
Object.defineProperty(fileOperation, "handle", {
|
||||
get() {
|
||||
return {
|
||||
path: path.resolve(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"test",
|
||||
"fixtures",
|
||||
"notion-markdown.zip"
|
||||
),
|
||||
cleanup: async () => {},
|
||||
};
|
||||
},
|
||||
});
|
||||
jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
|
||||
|
||||
const props = {
|
||||
fileOperationId: fileOperation.id,
|
||||
};
|
||||
|
||||
const task = new ImportNotionTask();
|
||||
const response = await task.perform(props);
|
||||
|
||||
expect(response.collections.size).toEqual(2);
|
||||
expect(response.documents.size).toEqual(6);
|
||||
expect(response.attachments.size).toEqual(1);
|
||||
|
||||
// Check that the image url was replaced in the text with a redirect
|
||||
const attachments = Array.from(response.attachments.values());
|
||||
const documents = Array.from(response.documents.values());
|
||||
expect(documents.map((d) => d.text).join("")).toContain(
|
||||
attachments[0].redirectUrl
|
||||
);
|
||||
});
|
||||
|
||||
it("should import successfully from a HTML export", async () => {
|
||||
const fileOperation = await buildFileOperation();
|
||||
Object.defineProperty(fileOperation, "handle", {
|
||||
get() {
|
||||
return {
|
||||
path: path.resolve(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"test",
|
||||
"fixtures",
|
||||
"notion-html.zip"
|
||||
),
|
||||
cleanup: async () => {},
|
||||
};
|
||||
},
|
||||
});
|
||||
jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
|
||||
|
||||
const props = {
|
||||
fileOperationId: fileOperation.id,
|
||||
};
|
||||
|
||||
const task = new ImportNotionTask();
|
||||
const response = await task.perform(props);
|
||||
|
||||
expect(response.collections.size).toEqual(2);
|
||||
expect(response.documents.size).toEqual(6);
|
||||
expect(response.attachments.size).toEqual(4);
|
||||
|
||||
// Check that the image url was replaced in the text with a redirect
|
||||
const attachments = Array.from(response.attachments.values());
|
||||
const attachment = attachments.find((att) =>
|
||||
att.key.endsWith("Screen_Shot_2022-04-21_at_2.23.26_PM.png")
|
||||
);
|
||||
|
||||
const documents = Array.from(response.documents.values());
|
||||
expect(documents.map((d) => d.text).join("")).toContain(
|
||||
attachment?.redirectUrl
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,354 +0,0 @@
|
||||
import path from "path";
|
||||
import fs from "fs-extra";
|
||||
import compact from "lodash/compact";
|
||||
import escapeRegExp from "lodash/escapeRegExp";
|
||||
import mime from "mime-types";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import documentImporter from "@server/commands/documentImporter";
|
||||
import { createContext } from "@server/context";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { FileOperation, User } from "@server/models";
|
||||
import { sequelize } from "@server/storage/database";
|
||||
import ImportHelper, { FileTreeNode } from "@server/utils/ImportHelper";
|
||||
import ImportTask, { StructuredImportData } from "./ImportTask";
|
||||
|
||||
export default class ImportNotionTask extends ImportTask {
|
||||
public async parseData(
|
||||
dirPath: string,
|
||||
fileOperation: FileOperation
|
||||
): Promise<StructuredImportData> {
|
||||
const tree = await ImportHelper.toFileTree(dirPath);
|
||||
if (!tree) {
|
||||
throw new Error("Could not find valid content in zip file");
|
||||
}
|
||||
|
||||
// New Notion exports have a single folder with the name of the export, we must skip this
|
||||
// folder and go directly to the children.
|
||||
let parsed;
|
||||
if (
|
||||
tree.children.length === 1 &&
|
||||
tree.children[0].children.find((child) => child.title === "index")
|
||||
) {
|
||||
parsed = await this.parseFileTree(
|
||||
fileOperation,
|
||||
tree.children[0].children.filter((child) => child.title !== "index")
|
||||
);
|
||||
} else {
|
||||
parsed = await this.parseFileTree(fileOperation, tree.children);
|
||||
}
|
||||
|
||||
if (parsed.documents.length === 0 && parsed.collections.length === 1) {
|
||||
const collection = parsed.collections[0];
|
||||
const collectionId = uuidv4();
|
||||
if (collection.description) {
|
||||
parsed.documents.push({
|
||||
title: collection.name,
|
||||
icon: collection.icon,
|
||||
color: collection.color,
|
||||
path: "",
|
||||
text: String(collection.description),
|
||||
id: collection.id,
|
||||
externalId: collection.externalId,
|
||||
mimeType: "text/html",
|
||||
collectionId,
|
||||
});
|
||||
}
|
||||
|
||||
collection.name = "Notion";
|
||||
collection.icon = undefined;
|
||||
collection.color = undefined;
|
||||
collection.externalId = undefined;
|
||||
collection.description = undefined;
|
||||
collection.id = collectionId;
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the file structure from zipAsFileTree into documents,
|
||||
* collections, and attachments.
|
||||
*
|
||||
* @param fileOperation The file operation
|
||||
* @param tree An array of FileTreeNode representing root files in the zip
|
||||
* @returns A StructuredImportData object
|
||||
*/
|
||||
private async parseFileTree(
|
||||
fileOperation: FileOperation,
|
||||
tree: FileTreeNode[]
|
||||
): Promise<StructuredImportData> {
|
||||
const user = await User.findByPk(fileOperation.userId, {
|
||||
rejectOnEmpty: true,
|
||||
});
|
||||
|
||||
const output: StructuredImportData = {
|
||||
collections: [],
|
||||
documents: [],
|
||||
attachments: [],
|
||||
};
|
||||
|
||||
const parseNodeChildren = async (
|
||||
children: FileTreeNode[],
|
||||
collectionId: string,
|
||||
parentDocumentId?: string
|
||||
): Promise<void> => {
|
||||
await Promise.all(
|
||||
children.map(async (child) => {
|
||||
// Ignore the CSV's for databases upfront
|
||||
if (child.path.endsWith(".csv")) {
|
||||
return;
|
||||
}
|
||||
|
||||
const id = uuidv4();
|
||||
const match = child.title.match(this.NotionUUIDRegex);
|
||||
const name = child.title.replace(this.NotionUUIDRegex, "");
|
||||
const externalId = match ? match[0].trim() : undefined;
|
||||
|
||||
// If it's not a text file we're going to treat it as an attachment.
|
||||
const mimeType = mime.lookup(child.name);
|
||||
const isDocument =
|
||||
mimeType === "text/markdown" ||
|
||||
mimeType === "text/plain" ||
|
||||
mimeType === "text/html";
|
||||
|
||||
// If it's not a document and not a folder, treat it as an attachment
|
||||
if (!isDocument && mimeType) {
|
||||
output.attachments.push({
|
||||
id,
|
||||
name: child.name,
|
||||
path: child.path,
|
||||
mimeType,
|
||||
buffer: () => fs.readFile(child.path),
|
||||
externalId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
Logger.debug("task", `Processing ${name} as ${mimeType}`);
|
||||
|
||||
const { title, icon, text } = await sequelize.transaction(
|
||||
async (transaction) =>
|
||||
documentImporter({
|
||||
mimeType: mimeType || "text/markdown",
|
||||
fileName: name,
|
||||
content:
|
||||
child.children.length > 0
|
||||
? ""
|
||||
: await fs.readFile(child.path, "utf8"),
|
||||
user,
|
||||
ctx: createContext({ user, transaction }),
|
||||
})
|
||||
);
|
||||
|
||||
const existingDocumentIndex = output.documents.findIndex(
|
||||
(doc) => doc.externalId === externalId
|
||||
);
|
||||
|
||||
const existingDocument = output.documents[existingDocumentIndex];
|
||||
|
||||
// If there is an existing document with the same externalId that means
|
||||
// we've already parsed either a folder or a file referencing the same
|
||||
// document, as such we should merge.
|
||||
if (existingDocument) {
|
||||
if (existingDocument.text === "") {
|
||||
output.documents[existingDocumentIndex].text = text;
|
||||
}
|
||||
|
||||
await parseNodeChildren(
|
||||
child.children,
|
||||
collectionId,
|
||||
existingDocument.id
|
||||
);
|
||||
} else {
|
||||
output.documents.push({
|
||||
id,
|
||||
title,
|
||||
icon,
|
||||
text,
|
||||
collectionId,
|
||||
parentDocumentId,
|
||||
path: child.path,
|
||||
mimeType: mimeType || "text/markdown",
|
||||
externalId,
|
||||
});
|
||||
await parseNodeChildren(child.children, collectionId, id);
|
||||
}
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const replaceInternalLinksAndImages = (text: string) => {
|
||||
// Find if there are any images in this document
|
||||
const imagesInText = this.parseImages(text);
|
||||
|
||||
for (const image of imagesInText) {
|
||||
const name = path.basename(image.src);
|
||||
const attachment = output.attachments.find(
|
||||
(att) =>
|
||||
att.path.endsWith(image.src) ||
|
||||
encodeURI(att.path).endsWith(image.src)
|
||||
);
|
||||
|
||||
if (!attachment) {
|
||||
if (!image.src.startsWith("http")) {
|
||||
Logger.info(
|
||||
"task",
|
||||
`Could not find referenced attachment with name ${name} and src ${image.src}`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
text = text.replace(
|
||||
new RegExp(escapeRegExp(image.src), "g"),
|
||||
`<<${attachment.id}>>`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// With Notion's HTML import, images sometimes come wrapped in anchor tags
|
||||
// This isn't supported in Outline's editor, so we need to strip them.
|
||||
text = text.replace(/\[!\[([^[]+)]/g, "![]");
|
||||
|
||||
// Find if there are any links in this document pointing to other documents
|
||||
const internalLinksInText = this.parseInternalLinks(text);
|
||||
|
||||
// For each link update to the standardized format of <<documentId>>
|
||||
// instead of a relative or absolute URL within the original zip file.
|
||||
for (const link of internalLinksInText) {
|
||||
const doc = output.documents.find(
|
||||
(doc) => doc.externalId === link.externalId
|
||||
);
|
||||
|
||||
if (!doc) {
|
||||
Logger.info(
|
||||
"task",
|
||||
`Could not find referenced document with externalId ${link.externalId}`
|
||||
);
|
||||
} else {
|
||||
text = text.replace(link.href, `<<${doc.id}>>`);
|
||||
}
|
||||
}
|
||||
|
||||
return text;
|
||||
};
|
||||
|
||||
// All nodes in the root level should become collections
|
||||
for (const node of tree) {
|
||||
const match = node.title.match(this.NotionUUIDRegex);
|
||||
const name = node.title.replace(this.NotionUUIDRegex, "");
|
||||
const externalId = match ? match[0].trim() : undefined;
|
||||
const mimeType = mime.lookup(node.name);
|
||||
|
||||
const existingCollectionIndex = output.collections.findIndex(
|
||||
(collection) => collection.externalId === externalId
|
||||
);
|
||||
const existingCollection = output.collections[existingCollectionIndex];
|
||||
const collectionId = existingCollection?.id || uuidv4();
|
||||
let description;
|
||||
|
||||
// Root level docs become the descriptions of collections
|
||||
if (
|
||||
mimeType === "text/markdown" ||
|
||||
mimeType === "text/plain" ||
|
||||
mimeType === "text/html"
|
||||
) {
|
||||
const { text } = await sequelize.transaction(async (transaction) =>
|
||||
documentImporter({
|
||||
mimeType,
|
||||
fileName: name,
|
||||
content: await fs.readFile(node.path, "utf8"),
|
||||
user,
|
||||
ctx: createContext({ user, transaction }),
|
||||
})
|
||||
);
|
||||
|
||||
description = text;
|
||||
} else if (node.children.length > 0) {
|
||||
await parseNodeChildren(node.children, collectionId);
|
||||
} else {
|
||||
Logger.debug("task", `Unhandled file in zip: ${node.path}`, {
|
||||
fileOperationId: fileOperation.id,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existingCollectionIndex !== -1) {
|
||||
if (description) {
|
||||
output.collections[existingCollectionIndex].description = description;
|
||||
}
|
||||
} else {
|
||||
output.collections.push({
|
||||
id: collectionId,
|
||||
name,
|
||||
description,
|
||||
externalId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const document of output.documents) {
|
||||
document.text = replaceInternalLinksAndImages(document.text);
|
||||
}
|
||||
|
||||
for (const collection of output.collections) {
|
||||
if (typeof collection.description === "string") {
|
||||
collection.description = replaceInternalLinksAndImages(
|
||||
collection.description
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts internal links from a markdown document, taking into account the
|
||||
* externalId of the document, which is part of the link title.
|
||||
*
|
||||
* @param text The markdown text to parse
|
||||
* @returns An array of internal links
|
||||
*/
|
||||
private parseInternalLinks(
|
||||
text: string
|
||||
): { title: string; href: string; externalId: string }[] {
|
||||
return compact(
|
||||
[...text.matchAll(this.NotionLinkRegex)].map((match) => ({
|
||||
title: match[1],
|
||||
href: match[2],
|
||||
externalId: match[3],
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts images from the markdown document
|
||||
*
|
||||
* @param text The markdown text to parse
|
||||
* @returns An array of internal links
|
||||
*/
|
||||
private parseImages(text: string): { alt: string; src: string }[] {
|
||||
return compact(
|
||||
[...text.matchAll(this.ImageRegex)].map((match) => ({
|
||||
alt: match[1],
|
||||
src: match[2],
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Regex to find markdown images of all types
|
||||
*/
|
||||
private ImageRegex =
|
||||
/!\[(?<alt>[^\][]*?)]\((?<filename>[^\][]*?)(?=“|\))“?(?<title>[^\][”]+)?”?\)/g;
|
||||
|
||||
/**
|
||||
* Regex to find markdown links containing ID's that look like UUID's with the
|
||||
* "-"'s removed, Notion's externalId format.
|
||||
*/
|
||||
private NotionLinkRegex = /\[([^[]+)]\((.*?([0-9a-fA-F]{32})\..*?)\)/g;
|
||||
|
||||
/**
|
||||
* Regex to find Notion document UUID's in the title of a document.
|
||||
*/
|
||||
private NotionUUIDRegex =
|
||||
/\s([0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}|[0-9a-fA-F]{32})$/;
|
||||
}
|
||||
@@ -597,6 +597,7 @@ router.post(
|
||||
createdById: user.id,
|
||||
},
|
||||
transaction,
|
||||
hooks: false,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import queryString from "query-string";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomElement } from "@shared/random";
|
||||
import { NotificationEventType } from "@shared/types";
|
||||
import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper";
|
||||
import {
|
||||
buildCollection,
|
||||
buildDocument,
|
||||
@@ -697,3 +698,40 @@ describe("#notifications.update_all", () => {
|
||||
expect(body.data.total).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#notifications.unsubscribe", () => {
|
||||
it("should allow unsubscribe with valid token", async () => {
|
||||
const user = await buildUser();
|
||||
const token = NotificationSettingsHelper.unsubscribeToken(
|
||||
user.id,
|
||||
NotificationEventType.UpdateDocument
|
||||
);
|
||||
|
||||
const res = await server.get(
|
||||
`/api/notifications.unsubscribe?userId=${user.id}&token=${token}&eventType=documents.update&follow=true`,
|
||||
{
|
||||
redirect: "manual",
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(302);
|
||||
expect(res.headers.get("location")).toContain(
|
||||
"/settings/notifications?success"
|
||||
);
|
||||
|
||||
const events = (await user.reload()).notificationSettings;
|
||||
expect(events).not.toContain("documents.update");
|
||||
});
|
||||
|
||||
it("should not allow unsubscribe with invalid token", async () => {
|
||||
const user = await buildUser();
|
||||
|
||||
const res = await server.get(
|
||||
`/api/notifications.unsubscribe?userId=${user.id}&token=invalid-token&eventType=documents.update&follow=true`,
|
||||
{
|
||||
redirect: "manual",
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(302);
|
||||
expect(res.headers.get("location")).toContain("?notice=invalid-auth");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,7 +53,7 @@ const handleUnsubscribe = async (
|
||||
});
|
||||
|
||||
user.setNotificationEventType(eventType, false);
|
||||
await user.save();
|
||||
await user.save({ transaction });
|
||||
ctx.redirect(`${user.team.url}/settings/notifications?success`);
|
||||
};
|
||||
|
||||
|
||||
Vendored
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,9 +0,0 @@
|
||||
import { generateAvatarUrl } from "./avatars";
|
||||
|
||||
it("should return clearbit url if available", async () => {
|
||||
const url = await generateAvatarUrl({
|
||||
id: "google",
|
||||
domain: "google.com",
|
||||
});
|
||||
expect(url).toBe("https://logo.clearbit.com/google.com");
|
||||
});
|
||||
@@ -1,28 +0,0 @@
|
||||
import crypto from "crypto";
|
||||
import fetch from "./fetch";
|
||||
|
||||
export async function generateAvatarUrl({
|
||||
id,
|
||||
domain,
|
||||
}: {
|
||||
id: string;
|
||||
domain?: string;
|
||||
}) {
|
||||
// attempt to get logo from Clearbit API. If one doesn't exist then
|
||||
// fall back to using tiley to generate a placeholder logo
|
||||
const hash = crypto.createHash("sha256");
|
||||
hash.update(id);
|
||||
let cbResponse, cbUrl;
|
||||
|
||||
if (domain) {
|
||||
cbUrl = `https://logo.clearbit.com/${domain}`;
|
||||
|
||||
try {
|
||||
cbResponse = await fetch(cbUrl);
|
||||
} catch (err) {
|
||||
// okay
|
||||
}
|
||||
}
|
||||
|
||||
return cbUrl && cbResponse && cbResponse.status === 200 ? cbUrl : null;
|
||||
}
|
||||
@@ -903,9 +903,6 @@
|
||||
"{{ count }} document imported_plural": "{{ count }} documents imported",
|
||||
"You can import a zip file that was previously exported from an Outline installation – collections, documents, and images will be imported. In Outline, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.": "You can import a zip file that was previously exported from an Outline installation – collections, documents, and images will be imported. In Outline, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.",
|
||||
"Drag and drop the zip file from the Markdown export option in {{appName}}, or click to upload": "Drag and drop the zip file from the Markdown export option in {{appName}}, or click to upload",
|
||||
"Where do I find the file?": "Where do I find the file?",
|
||||
"In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability.": "In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability.",
|
||||
"Drag and drop the zip file from Notion's HTML export option, or click to upload": "Drag and drop the zip file from Notion's HTML export option, or click to upload",
|
||||
"Last active": "Last active",
|
||||
"Guest": "Guest",
|
||||
"Shared by": "Shared by",
|
||||
|
||||
Reference in New Issue
Block a user