mirror of
https://github.com/outline/outline.git
synced 2026-06-13 19:35:02 +03:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5384ee8aae |
@@ -5,6 +5,7 @@ import { Redirect } from "react-router-dom";
|
||||
import useCurrentUser from "~/hooks/useCurrentUser";
|
||||
import useStores from "~/hooks/useStores";
|
||||
import { changeLanguage } from "~/utils/language";
|
||||
import { logoutPath } from "~/utils/routeHelpers";
|
||||
import LoadingIndicator from "./LoadingIndicator";
|
||||
|
||||
type Props = {
|
||||
@@ -32,7 +33,7 @@ const Authenticated = ({ children }: Props) => {
|
||||
}
|
||||
|
||||
void auth.logout(true);
|
||||
return <Redirect to="/" />;
|
||||
return <Redirect to={logoutPath()} />;
|
||||
};
|
||||
|
||||
export default observer(Authenticated);
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
import { AnimatePresence } from "framer-motion";
|
||||
import { observer } from "mobx-react";
|
||||
import * as React from "react";
|
||||
import {
|
||||
Switch,
|
||||
Route,
|
||||
useLocation,
|
||||
matchPath,
|
||||
Redirect,
|
||||
} from "react-router-dom";
|
||||
import { Switch, Route, useLocation, matchPath } from "react-router-dom";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import ErrorSuspended from "~/scenes/Errors/ErrorSuspended";
|
||||
import Layout from "~/components/Layout";
|
||||
@@ -16,7 +10,6 @@ import Sidebar from "~/components/Sidebar";
|
||||
import SidebarRight from "~/components/Sidebar/Right";
|
||||
import SettingsSidebar from "~/components/Sidebar/Settings";
|
||||
import useCurrentTeam from "~/hooks/useCurrentTeam";
|
||||
import { usePostLoginPath } from "~/hooks/useLastVisitedPath";
|
||||
import usePolicy from "~/hooks/usePolicy";
|
||||
import useStores from "~/hooks/useStores";
|
||||
import history from "~/utils/history";
|
||||
@@ -55,7 +48,6 @@ const AuthenticatedLayout: React.FC = ({ children }: Props) => {
|
||||
const can = usePolicy(ui.activeDocumentId);
|
||||
const canCollection = usePolicy(ui.activeCollectionId);
|
||||
const team = useCurrentTeam();
|
||||
const [spendPostLoginPath] = usePostLoginPath();
|
||||
|
||||
const goToSearch = (ev: KeyboardEvent) => {
|
||||
if (!ev.metaKey && !ev.ctrlKey) {
|
||||
@@ -80,11 +72,6 @@ const AuthenticatedLayout: React.FC = ({ children }: Props) => {
|
||||
return <ErrorSuspended />;
|
||||
}
|
||||
|
||||
const postLoginPath = spendPostLoginPath();
|
||||
if (postLoginPath) {
|
||||
return <Redirect to={postLoginPath} />;
|
||||
}
|
||||
|
||||
const sidebar = (
|
||||
<Fade>
|
||||
<Switch>
|
||||
|
||||
@@ -45,10 +45,6 @@ export const NativeInput = styled.input<{
|
||||
${ellipsis()}
|
||||
${undraggableOnDesktop()}
|
||||
|
||||
&[readOnly] {
|
||||
color: ${s("textSecondary")};
|
||||
}
|
||||
|
||||
&:disabled,
|
||||
&::placeholder {
|
||||
color: ${s("placeholder")};
|
||||
@@ -130,14 +126,13 @@ export interface Props
|
||||
React.InputHTMLAttributes<HTMLInputElement | HTMLTextAreaElement>,
|
||||
"prefix"
|
||||
> {
|
||||
type?: "text" | "email" | "checkbox" | "search" | "textarea" | "password";
|
||||
type?: "text" | "email" | "checkbox" | "search" | "textarea";
|
||||
labelHidden?: boolean;
|
||||
label?: string;
|
||||
flex?: boolean;
|
||||
short?: boolean;
|
||||
margin?: string | number;
|
||||
error?: string;
|
||||
rows?: number;
|
||||
/** Optional component that appears inside the input before the textarea and any icon */
|
||||
prefix?: React.ReactNode;
|
||||
/** Optional icon that appears inside the input before the textarea */
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { observer } from "mobx-react";
|
||||
import * as React from "react";
|
||||
import styled from "styled-components";
|
||||
import { s } from "@shared/styles";
|
||||
import Flex from "~/components/Flex";
|
||||
|
||||
type Props = {
|
||||
children?: React.ReactNode;
|
||||
label: React.ReactNode | string;
|
||||
};
|
||||
|
||||
const Labeled: React.FC<Props> = ({ label, children, ...props }: Props) => (
|
||||
<Flex column {...props}>
|
||||
<Label>{label}</Label>
|
||||
{children}
|
||||
</Flex>
|
||||
);
|
||||
|
||||
export const Label = styled(Flex)`
|
||||
font-weight: 500;
|
||||
padding-bottom: 4px;
|
||||
display: inline-block;
|
||||
color: ${s("text")};
|
||||
`;
|
||||
|
||||
export default observer(Labeled);
|
||||
+8
-1
@@ -2,7 +2,7 @@ import { observer } from "mobx-react";
|
||||
import { HomeIcon } from "outline-icons";
|
||||
import * as React from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Switch, Route } from "react-router-dom";
|
||||
import { Switch, Route, Redirect } from "react-router-dom";
|
||||
import styled from "styled-components";
|
||||
import { s } from "@shared/styles";
|
||||
import { Action } from "~/components/Actions";
|
||||
@@ -18,6 +18,7 @@ import Tab from "~/components/Tab";
|
||||
import Tabs from "~/components/Tabs";
|
||||
import useCurrentTeam from "~/hooks/useCurrentTeam";
|
||||
import useCurrentUser from "~/hooks/useCurrentUser";
|
||||
import { usePostLoginPath } from "~/hooks/useLastVisitedPath";
|
||||
import { usePinnedDocuments } from "~/hooks/usePinnedDocuments";
|
||||
import usePolicy from "~/hooks/usePolicy";
|
||||
import useStores from "~/hooks/useStores";
|
||||
@@ -28,10 +29,16 @@ function Home() {
|
||||
const team = useCurrentTeam();
|
||||
const user = useCurrentUser();
|
||||
const { t } = useTranslation();
|
||||
const [spendPostLoginPath] = usePostLoginPath();
|
||||
const userId = user?.id;
|
||||
const { pins, count } = usePinnedDocuments("home");
|
||||
const can = usePolicy(team);
|
||||
|
||||
const postLoginPath = spendPostLoginPath();
|
||||
if (postLoginPath) {
|
||||
return <Redirect to={postLoginPath} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<Scene
|
||||
icon={<HomeIcon />}
|
||||
|
||||
@@ -9,7 +9,7 @@ import { s } from "@shared/styles";
|
||||
import { AttachmentPreset } from "@shared/types";
|
||||
import { AttachmentValidation } from "@shared/validations";
|
||||
import RootStore from "~/stores/RootStore";
|
||||
import ButtonLarge from "~/components/ButtonLarge";
|
||||
import Button from "~/components/Button";
|
||||
import Flex from "~/components/Flex";
|
||||
import LoadingIndicator from "~/components/LoadingIndicator";
|
||||
import Modal from "~/components/Modal";
|
||||
@@ -95,14 +95,8 @@ class ImageUpload extends React.Component<RootStore & Props> {
|
||||
|
||||
renderCropping() {
|
||||
const { ui, submitText } = this.props;
|
||||
|
||||
return (
|
||||
<Modal
|
||||
onRequestClose={this.handleClose}
|
||||
fullscreen={false}
|
||||
title={<> </>}
|
||||
isOpen
|
||||
>
|
||||
<Modal isOpen onRequestClose={this.handleClose} title="">
|
||||
<Flex auto column align="center" justify="center">
|
||||
{this.isUploading && <LoadingIndicator />}
|
||||
<AvatarEditorContainer>
|
||||
@@ -128,14 +122,9 @@ class ImageUpload extends React.Component<RootStore & Props> {
|
||||
defaultValue="1"
|
||||
onChange={this.handleZoom}
|
||||
/>
|
||||
<br />
|
||||
<ButtonLarge
|
||||
fullwidth
|
||||
onClick={this.handleCrop}
|
||||
disabled={this.isUploading}
|
||||
>
|
||||
<CropButton onClick={this.handleCrop} disabled={this.isUploading}>
|
||||
{this.isUploading ? "Uploading…" : submitText}
|
||||
</ButtonLarge>
|
||||
</CropButton>
|
||||
</Flex>
|
||||
</Modal>
|
||||
);
|
||||
@@ -191,4 +180,8 @@ const RangeInput = styled.input`
|
||||
}
|
||||
`;
|
||||
|
||||
const CropButton = styled(Button)`
|
||||
width: 300px;
|
||||
`;
|
||||
|
||||
export default withStores(ImageUpload);
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import * as React from "react";
|
||||
import { Trans, useTranslation } from "react-i18next";
|
||||
import { FileOperationFormat } from "@shared/types";
|
||||
import useStores from "~/hooks/useStores";
|
||||
import DropToImport from "./DropToImport";
|
||||
import HelpDisclosure from "./HelpDisclosure";
|
||||
|
||||
function ImportNotionDialog() {
|
||||
const { t } = useTranslation();
|
||||
const { dialogs } = useStores();
|
||||
|
||||
return (
|
||||
<>
|
||||
<HelpDisclosure title={<Trans>Where do I find the file?</Trans>}>
|
||||
<Trans
|
||||
defaults="In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability."
|
||||
components={{
|
||||
em: <strong />,
|
||||
}}
|
||||
/>
|
||||
</HelpDisclosure>
|
||||
<DropToImport
|
||||
onSubmit={dialogs.closeAllModals}
|
||||
format={FileOperationFormat.Notion}
|
||||
>
|
||||
<>
|
||||
{t(
|
||||
`Drag and drop the zip file from Notion's HTML export option, or click to upload`
|
||||
)}
|
||||
</>
|
||||
</DropToImport>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default ImportNotionDialog;
|
||||
@@ -306,7 +306,7 @@ export default class AuthStore extends Store<Team> {
|
||||
// if this logout was forced from an authenticated route then
|
||||
// save the current path so we can go back there once signed in
|
||||
if (savePath) {
|
||||
setPostLoginPath(window.location.pathname + window.location.search);
|
||||
setPostLoginPath(window.location.pathname);
|
||||
}
|
||||
|
||||
if (tryRevokingToken) {
|
||||
|
||||
+3
-5
@@ -49,10 +49,8 @@ export function redirectTo(url: string) {
|
||||
/**
|
||||
* Check if the path is a valid path for redirect after login.
|
||||
*
|
||||
* @param input A path potentially including query string
|
||||
* @param path
|
||||
* @returns boolean indicating if the path is a valid redirect
|
||||
*/
|
||||
export const isAllowedLoginRedirect = (input: string) => {
|
||||
const path = input.split("?")[0];
|
||||
return !["/", "/create", "/home", "/logout", "/auth/"].includes(path);
|
||||
};
|
||||
export const isAllowedLoginRedirect = (path: string) =>
|
||||
!["/", "/create", "/home", "/logout", "/auth/"].includes(path);
|
||||
|
||||
+10
-10
@@ -48,11 +48,11 @@
|
||||
"> 0.25%, not dead"
|
||||
],
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.777.0",
|
||||
"@aws-sdk/lib-storage": "3.777.0",
|
||||
"@aws-sdk/s3-presigned-post": "3.777.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.777.0",
|
||||
"@aws-sdk/signature-v4-crt": "^3.775.0",
|
||||
"@aws-sdk/client-s3": "3.774.0",
|
||||
"@aws-sdk/lib-storage": "3.774.0",
|
||||
"@aws-sdk/s3-presigned-post": "3.774.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.774.0",
|
||||
"@aws-sdk/signature-v4-crt": "^3.774.0",
|
||||
"@babel/core": "^7.26.10",
|
||||
"@babel/plugin-proposal-decorators": "^7.25.9",
|
||||
"@babel/plugin-transform-class-properties": "^7.25.9",
|
||||
@@ -89,7 +89,7 @@
|
||||
"@sentry/node": "^7.120.3",
|
||||
"@sentry/react": "^7.120.3",
|
||||
"@tanstack/react-table": "^8.20.6",
|
||||
"@tanstack/react-virtual": "^3.13.6",
|
||||
"@tanstack/react-virtual": "^3.11.3",
|
||||
"@tippyjs/react": "^4.2.6",
|
||||
"@types/form-data": "^2.5.2",
|
||||
"@types/mailparser": "^3.4.5",
|
||||
@@ -185,8 +185,8 @@
|
||||
"prosemirror-history": "^1.4.1",
|
||||
"prosemirror-inputrules": "^1.4.0",
|
||||
"prosemirror-keymap": "^1.2.2",
|
||||
"prosemirror-markdown": "^1.13.2",
|
||||
"prosemirror-model": "^1.25.0",
|
||||
"prosemirror-markdown": "^1.13.1",
|
||||
"prosemirror-model": "^1.24.0",
|
||||
"prosemirror-schema-list": "^1.4.1",
|
||||
"prosemirror-state": "^1.4.3",
|
||||
"prosemirror-tables": "^1.6.4",
|
||||
@@ -248,7 +248,7 @@
|
||||
"uuid": "^8.3.2",
|
||||
"validator": "13.12.0",
|
||||
"vaul": "^1.1.2",
|
||||
"vite": "^5.4.16",
|
||||
"vite": "^5.4.15",
|
||||
"vite-plugin-pwa": "^0.20.3",
|
||||
"winston": "^3.17.0",
|
||||
"ws": "^7.5.10",
|
||||
@@ -263,7 +263,7 @@
|
||||
"@babel/cli": "^7.27.0",
|
||||
"@babel/preset-typescript": "^7.27.0",
|
||||
"@faker-js/faker": "^8.4.1",
|
||||
"@relative-ci/agent": "^4.3.0",
|
||||
"@relative-ci/agent": "^4.2.14",
|
||||
"@testing-library/react": "^12.0.0",
|
||||
"@types/addressparser": "^1.0.3",
|
||||
"@types/body-scroll-lock": "^3.1.2",
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { APIResponseError, APIErrorCode } from "@notionhq/client";
|
||||
import { ImportTaskInput, ImportTaskOutput } from "@shared/schema";
|
||||
import { IntegrationService, ProsemirrorDoc } from "@shared/types";
|
||||
import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { Integration } from "@server/models";
|
||||
import ImportTask from "@server/models/ImportTask";
|
||||
import APIImportTask, {
|
||||
@@ -41,10 +39,7 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
importTask.input.map(async (item) => this.processPage({ item, client }))
|
||||
);
|
||||
|
||||
// Filter out any null results (from pages/databases that couldn't be accessed)
|
||||
const validParsedPages = parsedPages.filter(Boolean) as ParsePageOutput[];
|
||||
|
||||
const taskOutput: ImportTaskOutput = validParsedPages.map((parsedPage) => ({
|
||||
const taskOutput: ImportTaskOutput = parsedPages.map((parsedPage) => ({
|
||||
externalId: parsedPage.externalId,
|
||||
title: parsedPage.title,
|
||||
emoji: parsedPage.emoji,
|
||||
@@ -55,7 +50,7 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
}));
|
||||
|
||||
const childTasksInput: ImportTaskInput<IntegrationService.Notion> =
|
||||
validParsedPages.flatMap((parsedPage) =>
|
||||
parsedPages.flatMap((parsedPage) =>
|
||||
parsedPage.children.map((childPage) => ({
|
||||
type: childPage.type,
|
||||
externalId: childPage.externalId,
|
||||
@@ -93,55 +88,36 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
}: {
|
||||
item: ImportTaskInput<IntegrationService.Notion>[number];
|
||||
client: NotionClient;
|
||||
}): Promise<ParsePageOutput | null> {
|
||||
}): Promise<ParsePageOutput> {
|
||||
const collectionExternalId = item.collectionExternalId ?? item.externalId;
|
||||
|
||||
try {
|
||||
// Convert Notion database to an empty page with "pages in database" as its children.
|
||||
if (item.type === PageType.Database) {
|
||||
const { pages, ...databaseInfo } = await client.fetchDatabase(
|
||||
item.externalId
|
||||
);
|
||||
|
||||
return {
|
||||
...databaseInfo,
|
||||
externalId: item.externalId,
|
||||
content: ProsemirrorHelper.getEmptyDocument() as ProsemirrorDoc,
|
||||
collectionExternalId,
|
||||
children: pages.map((page) => ({
|
||||
type: page.type,
|
||||
externalId: page.id,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const { blocks, ...pageInfo } = await client.fetchPage(item.externalId);
|
||||
// Convert Notion database to an empty page with "pages in database" as its children.
|
||||
if (item.type === PageType.Database) {
|
||||
const { pages, ...databaseInfo } = await client.fetchDatabase(
|
||||
item.externalId
|
||||
);
|
||||
|
||||
return {
|
||||
...pageInfo,
|
||||
...databaseInfo,
|
||||
externalId: item.externalId,
|
||||
content: NotionConverter.page({ children: blocks } as NotionPage),
|
||||
content: ProsemirrorHelper.getEmptyDocument() as ProsemirrorDoc,
|
||||
collectionExternalId,
|
||||
children: this.parseChildPages(blocks),
|
||||
children: pages.map((page) => ({
|
||||
type: page.type,
|
||||
externalId: page.id,
|
||||
})),
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof APIResponseError) {
|
||||
// Skip this page/database if it's not found or not accessible
|
||||
if (
|
||||
error.code === APIErrorCode.ObjectNotFound ||
|
||||
error.code === APIErrorCode.Unauthorized
|
||||
) {
|
||||
Logger.warn(
|
||||
`Skipping Notion ${
|
||||
item.type === PageType.Database ? "database" : "page"
|
||||
} ${item.externalId} - Error code: ${error.code} - ${error.message}`
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
// Re-throw other errors to be handled by the parent try/catch
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { blocks, ...pageInfo } = await client.fetchPage(item.externalId);
|
||||
|
||||
return {
|
||||
...pageInfo,
|
||||
externalId: item.externalId,
|
||||
content: NotionConverter.page({ children: blocks } as NotionPage),
|
||||
collectionExternalId,
|
||||
children: this.parseChildPages(blocks),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
import { Server } from "@hocuspocus/server";
|
||||
import WebSocket from "ws";
|
||||
import EDITOR_VERSION from "@shared/editor/version";
|
||||
import { sleep } from "@server/utils/timers";
|
||||
import { ConnectionLimitExtension } from "./ConnectionLimitExtension";
|
||||
import { EditorVersionExtension } from "./EditorVersionExtension";
|
||||
|
||||
jest.mock("@server/env", () => ({
|
||||
COLLABORATION_MAX_CLIENTS_PER_DOCUMENT: 2,
|
||||
}));
|
||||
|
||||
describe("ConnectionLimitExtension", () => {
|
||||
let server: typeof Server;
|
||||
let extension: ConnectionLimitExtension;
|
||||
const port = 12345;
|
||||
const url = `ws://localhost:${port}`;
|
||||
const documentName = "test";
|
||||
|
||||
beforeEach(async () => {
|
||||
extension = new ConnectionLimitExtension();
|
||||
server = Server.configure({
|
||||
port,
|
||||
extensions: [extension, new EditorVersionExtension()],
|
||||
});
|
||||
await server.listen();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await server.destroy();
|
||||
});
|
||||
|
||||
const getConnections = () =>
|
||||
extension.connectionsByDocument.get(documentName)?.size ?? 0;
|
||||
|
||||
const createWebSocket = (editorVersion = EDITOR_VERSION) =>
|
||||
new Promise<WebSocket>((resolve, reject) => {
|
||||
const ws = new WebSocket(
|
||||
`${url}/${documentName}?editorVersion=${editorVersion}`
|
||||
);
|
||||
ws.on("open", () => resolve(ws));
|
||||
ws.on("error", reject);
|
||||
});
|
||||
|
||||
it("should allow connections within limit", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
expect(ws1.readyState).toBe(WebSocket.OPEN);
|
||||
expect(ws2.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should close connections exceeding limit", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
const ws3 = await createWebSocket();
|
||||
await sleep(250);
|
||||
|
||||
expect(ws3.readyState).toBe(WebSocket.CLOSED);
|
||||
expect(ws2.readyState).toBe(WebSocket.OPEN);
|
||||
expect(ws1.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should handle connections closed by other extensions", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
|
||||
// Create a connection that will be closed by the EditorVersionExtension
|
||||
const ws2 = await createWebSocket("1.0.0");
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should allow new connection after disconnect", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
ws1.close();
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(1);
|
||||
|
||||
const ws3 = await createWebSocket();
|
||||
expect(ws3.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws2.close();
|
||||
ws3.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,8 @@
|
||||
import {
|
||||
Extension,
|
||||
connectedPayload,
|
||||
onConnectPayload,
|
||||
onDisconnectPayload,
|
||||
} from "@hocuspocus/server";
|
||||
import pluralize from "pluralize";
|
||||
import { TooManyConnections } from "@shared/collaboration/CloseEvents";
|
||||
import env from "@server/env";
|
||||
import Logger from "@server/logging/Logger";
|
||||
@@ -16,7 +14,7 @@ export class ConnectionLimitExtension implements Extension {
|
||||
/**
|
||||
* Map of documentId -> connection count
|
||||
*/
|
||||
public connectionsByDocument: Map<string, Set<string>> = new Map();
|
||||
connectionsByDocument: Map<string, Set<string>> = new Map();
|
||||
|
||||
/**
|
||||
* On disconnect hook
|
||||
@@ -36,30 +34,23 @@ export class ConnectionLimitExtension implements Extension {
|
||||
}
|
||||
}
|
||||
|
||||
const connectionCount = connections?.size ?? 0;
|
||||
Logger.debug(
|
||||
"multiplayer",
|
||||
`${connectionCount} ${pluralize(
|
||||
"connection",
|
||||
connectionCount
|
||||
)} to "${documentName}"`
|
||||
`${connections?.size} connections to "${documentName}"`
|
||||
);
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
/**
|
||||
* onConnect hook is called when a new connection has been established.
|
||||
* This is where we can check if the document has reached the maximum number of
|
||||
* connections and reject the connection if it has.
|
||||
* On connect hook
|
||||
*
|
||||
* @param data The onConnect payload
|
||||
* @returns Promise, resolving will allow the connection, rejecting will drop.
|
||||
* @param data The connect payload
|
||||
* @returns Promise, resolving will allow the connection, rejecting will drop it
|
||||
*/
|
||||
onConnect({ documentName }: withContext<onConnectPayload>) {
|
||||
onConnect({ documentName, socketId }: withContext<onConnectPayload>) {
|
||||
const connections =
|
||||
this.connectionsByDocument.get(documentName) || new Set();
|
||||
|
||||
if (connections?.size >= env.COLLABORATION_MAX_CLIENTS_PER_DOCUMENT) {
|
||||
Logger.info(
|
||||
"multiplayer",
|
||||
@@ -70,30 +61,12 @@ export class ConnectionLimitExtension implements Extension {
|
||||
return Promise.reject(TooManyConnections);
|
||||
}
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
/**
|
||||
* Connected hook is called after a new connection has been established.
|
||||
* We can safely update the connection count for the document.
|
||||
*
|
||||
* @param data The onConnect payload
|
||||
* @returns Promise
|
||||
*/
|
||||
connected({ documentName, socketId }: withContext<connectedPayload>) {
|
||||
const connections =
|
||||
this.connectionsByDocument.get(documentName) || new Set();
|
||||
|
||||
connections.add(socketId);
|
||||
this.connectionsByDocument.set(documentName, connections);
|
||||
const connectionCount = connections.size ?? 0;
|
||||
|
||||
Logger.debug(
|
||||
"multiplayer",
|
||||
`${connectionCount} ${pluralize(
|
||||
"connection",
|
||||
connectionCount
|
||||
)} to "${documentName}"`
|
||||
`${connections.size} connections to "${documentName}"`
|
||||
);
|
||||
|
||||
return Promise.resolve();
|
||||
|
||||
@@ -3,6 +3,7 @@ import slugify from "slugify";
|
||||
import { RESERVED_SUBDOMAINS } from "@shared/utils/domains";
|
||||
import { traceFunction } from "@server/logging/tracing";
|
||||
import { Team, Event } from "@server/models";
|
||||
import { generateAvatarUrl } from "@server/utils/avatars";
|
||||
|
||||
type Props = {
|
||||
/** The displayed name of the team */
|
||||
@@ -28,14 +29,20 @@ type Props = {
|
||||
|
||||
async function teamCreator({
|
||||
name,
|
||||
domain,
|
||||
subdomain,
|
||||
avatarUrl,
|
||||
authenticationProviders,
|
||||
ip,
|
||||
transaction,
|
||||
}: Props): Promise<Team> {
|
||||
if (!avatarUrl?.startsWith("http")) {
|
||||
avatarUrl = null;
|
||||
// If the service did not provide a logo/avatar then we attempt to generate
|
||||
// one via ClearBit, or fallback to colored initials in worst case scenario
|
||||
if (!avatarUrl || !avatarUrl.startsWith("http")) {
|
||||
avatarUrl = await generateAvatarUrl({
|
||||
domain,
|
||||
id: subdomain,
|
||||
});
|
||||
}
|
||||
|
||||
const team = await Team.create(
|
||||
|
||||
@@ -192,7 +192,7 @@ describe("userProvisioner", () => {
|
||||
|
||||
it("should prefer isAdmin argument over defaultUserRole", async () => {
|
||||
const team = await buildTeam({
|
||||
defaultUserRole: UserRole.Viewer,
|
||||
defaultUserRole: "viewer",
|
||||
});
|
||||
const authenticationProviders = await team.$get("authenticationProviders");
|
||||
const authenticationProvider = authenticationProviders[0];
|
||||
|
||||
@@ -86,8 +86,7 @@ export default class DocumentPublishedOrUpdatedEmail extends BaseEmail<
|
||||
}
|
||||
return;
|
||||
},
|
||||
30,
|
||||
10000
|
||||
30
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -3,13 +3,14 @@ import compact from "lodash/compact";
|
||||
import flatten from "lodash/flatten";
|
||||
import isEqual from "lodash/isEqual";
|
||||
import uniq from "lodash/uniq";
|
||||
import { Node, DOMSerializer, Fragment } from "prosemirror-model";
|
||||
import { Node, DOMSerializer, Fragment, Mark } from "prosemirror-model";
|
||||
import * as React from "react";
|
||||
import { renderToString } from "react-dom/server";
|
||||
import styled, { ServerStyleSheet, ThemeProvider } from "styled-components";
|
||||
import { prosemirrorToYDoc } from "y-prosemirror";
|
||||
import * as Y from "yjs";
|
||||
import EditorContainer from "@shared/editor/components/Styles";
|
||||
import embeds from "@shared/editor/embeds";
|
||||
import GlobalStyles from "@shared/styles/globals";
|
||||
import light from "@shared/styles/theme";
|
||||
import { MentionType, ProsemirrorData } from "@shared/types";
|
||||
@@ -60,7 +61,47 @@ export class ProsemirrorHelper {
|
||||
);
|
||||
}
|
||||
|
||||
const node = parser.parse(input);
|
||||
let node = parser.parse(input);
|
||||
|
||||
// in the editor embeds are created at runtime by converting links into
|
||||
// embeds where they match.Because we're converting to a CRDT structure on
|
||||
// the server we need to mimic this behavior.
|
||||
function urlsToEmbeds(node: Node): Node {
|
||||
if (node.type.name === "paragraph") {
|
||||
for (const textNode of node.content.content) {
|
||||
for (const embed of embeds) {
|
||||
if (
|
||||
textNode.text &&
|
||||
textNode.marks.some(
|
||||
(m: Mark) =>
|
||||
m.type.name === "link" && m.attrs.href === textNode.text
|
||||
) &&
|
||||
embed.matcher(textNode.text)
|
||||
) {
|
||||
return schema.nodes.embed.createAndFill({
|
||||
href: textNode.text,
|
||||
}) as Node;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (node.content) {
|
||||
const contentAsArray =
|
||||
node.content instanceof Fragment
|
||||
? node.content.content
|
||||
: node.content;
|
||||
// @ts-expect-error content
|
||||
node.content = Fragment.fromArray(contentAsArray.map(urlsToEmbeds));
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
if (node) {
|
||||
node = urlsToEmbeds(node);
|
||||
}
|
||||
|
||||
return node ? prosemirrorToYDoc(node, fieldName) : new Y.Doc();
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import ExportJSONTask from "../tasks/ExportJSONTask";
|
||||
import ExportMarkdownZipTask from "../tasks/ExportMarkdownZipTask";
|
||||
import ImportJSONTask from "../tasks/ImportJSONTask";
|
||||
import ImportMarkdownZipTask from "../tasks/ImportMarkdownZipTask";
|
||||
import ImportNotionTask from "../tasks/ImportNotionTask";
|
||||
import BaseProcessor from "./BaseProcessor";
|
||||
|
||||
export default class FileOperationCreatedProcessor extends BaseProcessor {
|
||||
@@ -24,6 +25,11 @@ export default class FileOperationCreatedProcessor extends BaseProcessor {
|
||||
fileOperationId: event.modelId,
|
||||
});
|
||||
break;
|
||||
case FileOperationFormat.Notion:
|
||||
await ImportNotionTask.schedule({
|
||||
fileOperationId: event.modelId,
|
||||
});
|
||||
break;
|
||||
case FileOperationFormat.JSON:
|
||||
await ImportJSONTask.schedule({
|
||||
fileOperationId: event.modelId,
|
||||
|
||||
@@ -3,12 +3,7 @@ import chunk from "lodash/chunk";
|
||||
import keyBy from "lodash/keyBy";
|
||||
import truncate from "lodash/truncate";
|
||||
import { Fragment, Node } from "prosemirror-model";
|
||||
import {
|
||||
CreateOptions,
|
||||
CreationAttributes,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from "sequelize";
|
||||
import { CreateOptions, CreationAttributes, Transaction } from "sequelize";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomElement } from "@shared/random";
|
||||
import { ImportInput, ImportTaskInput } from "@shared/schema";
|
||||
@@ -159,59 +154,45 @@ export default abstract class ImportsProcessor<
|
||||
* @returns Promise that resolves when mapping and persistence is completed.
|
||||
*/
|
||||
private async onProcessed(importModel: Import<T>, transaction: Transaction) {
|
||||
try {
|
||||
const { collections } = await this.createCollectionsAndDocuments({
|
||||
importModel,
|
||||
transaction,
|
||||
});
|
||||
const { collections } = await this.createCollectionsAndDocuments({
|
||||
importModel,
|
||||
transaction,
|
||||
});
|
||||
|
||||
// Once all collections and documents are created, update collection's document structure.
|
||||
// This ensures the root documents have the whole subtree available in the structure.
|
||||
for (const collection of collections) {
|
||||
await Document.unscoped().findAllInBatches<Document>(
|
||||
{
|
||||
where: { parentDocumentId: null, collectionId: collection.id },
|
||||
order: [
|
||||
["createdAt", "DESC"],
|
||||
["id", "ASC"],
|
||||
],
|
||||
transaction,
|
||||
},
|
||||
async (documents) => {
|
||||
for (const document of documents) {
|
||||
await collection.addDocumentToStructure(document, 0, {
|
||||
save: false,
|
||||
silent: true,
|
||||
transaction,
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
await collection.save({ silent: true, transaction });
|
||||
}
|
||||
|
||||
importModel.state = ImportState.Completed;
|
||||
importModel.error = null; // unset any error from previous attempts.
|
||||
await importModel.saveWithCtx(
|
||||
createContext({
|
||||
user: importModel.createdBy,
|
||||
// Once all collections and documents are created, update collection's document structure.
|
||||
// This ensures the root documents have the whole subtree available in the structure.
|
||||
for (const collection of collections) {
|
||||
await Document.unscoped().findAllInBatches<Document>(
|
||||
{
|
||||
where: { parentDocumentId: null, collectionId: collection.id },
|
||||
order: [
|
||||
["createdAt", "DESC"],
|
||||
["id", "ASC"],
|
||||
],
|
||||
transaction,
|
||||
})
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof UniqueConstraintError) {
|
||||
Logger.error(
|
||||
"ImportsProcessor persistence failed due to unique constraint error",
|
||||
err,
|
||||
{
|
||||
fields: err.fields,
|
||||
},
|
||||
async (documents) => {
|
||||
for (const document of documents) {
|
||||
await collection.addDocumentToStructure(document, 0, {
|
||||
save: false,
|
||||
silent: true,
|
||||
transaction,
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
throw err;
|
||||
await collection.save({ silent: true, transaction });
|
||||
}
|
||||
|
||||
importModel.state = ImportState.Completed;
|
||||
importModel.error = null; // unset any error from previous attempts.
|
||||
await importModel.saveWithCtx(
|
||||
createContext({
|
||||
user: importModel.createdBy,
|
||||
transaction,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -323,15 +304,6 @@ export default abstract class ImportsProcessor<
|
||||
|
||||
const output = outputMap[externalId];
|
||||
|
||||
// Skip this item if it has no output (likely due to an error during processing)
|
||||
if (!output) {
|
||||
Logger.debug(
|
||||
"processor",
|
||||
`Skipping item with no output: ${externalId}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const collectionItem = importInput[externalId];
|
||||
|
||||
const attachments = await Attachment.findAll({
|
||||
@@ -472,7 +444,7 @@ export default abstract class ImportsProcessor<
|
||||
importInput: Record<string, ImportInput<any>[number]>;
|
||||
actorId: string;
|
||||
}): ProsemirrorDoc {
|
||||
// special case when the doc content is empty.
|
||||
// special case when the doc content is empty
|
||||
if (!content.content.length) {
|
||||
return content;
|
||||
}
|
||||
|
||||
@@ -170,7 +170,7 @@ export default abstract class APIImportTask<
|
||||
await importTask.save({ transaction });
|
||||
|
||||
const associatedImport = importTask.import;
|
||||
associatedImport.documentCount += taskOutputWithReplacements.length;
|
||||
associatedImport.documentCount += importTask.input.length;
|
||||
await associatedImport.saveWithCtx(
|
||||
createContext({
|
||||
user: associatedImport.createdBy,
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
/* eslint-disable @typescript-eslint/no-empty-function */
|
||||
import path from "path";
|
||||
import { FileOperation } from "@server/models";
|
||||
import { buildFileOperation } from "@server/test/factories";
|
||||
import ImportNotionTask from "./ImportNotionTask";
|
||||
|
||||
describe("ImportNotionTask", () => {
|
||||
it("should import successfully from a Markdown export", async () => {
|
||||
const fileOperation = await buildFileOperation();
|
||||
Object.defineProperty(fileOperation, "handle", {
|
||||
get() {
|
||||
return {
|
||||
path: path.resolve(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"test",
|
||||
"fixtures",
|
||||
"notion-markdown.zip"
|
||||
),
|
||||
cleanup: async () => {},
|
||||
};
|
||||
},
|
||||
});
|
||||
jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
|
||||
|
||||
const props = {
|
||||
fileOperationId: fileOperation.id,
|
||||
};
|
||||
|
||||
const task = new ImportNotionTask();
|
||||
const response = await task.perform(props);
|
||||
|
||||
expect(response.collections.size).toEqual(2);
|
||||
expect(response.documents.size).toEqual(6);
|
||||
expect(response.attachments.size).toEqual(1);
|
||||
|
||||
// Check that the image url was replaced in the text with a redirect
|
||||
const attachments = Array.from(response.attachments.values());
|
||||
const documents = Array.from(response.documents.values());
|
||||
expect(documents.map((d) => d.text).join("")).toContain(
|
||||
attachments[0].redirectUrl
|
||||
);
|
||||
});
|
||||
|
||||
it("should import successfully from a HTML export", async () => {
|
||||
const fileOperation = await buildFileOperation();
|
||||
Object.defineProperty(fileOperation, "handle", {
|
||||
get() {
|
||||
return {
|
||||
path: path.resolve(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"test",
|
||||
"fixtures",
|
||||
"notion-html.zip"
|
||||
),
|
||||
cleanup: async () => {},
|
||||
};
|
||||
},
|
||||
});
|
||||
jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
|
||||
|
||||
const props = {
|
||||
fileOperationId: fileOperation.id,
|
||||
};
|
||||
|
||||
const task = new ImportNotionTask();
|
||||
const response = await task.perform(props);
|
||||
|
||||
expect(response.collections.size).toEqual(2);
|
||||
expect(response.documents.size).toEqual(6);
|
||||
expect(response.attachments.size).toEqual(4);
|
||||
|
||||
// Check that the image url was replaced in the text with a redirect
|
||||
const attachments = Array.from(response.attachments.values());
|
||||
const attachment = attachments.find((att) =>
|
||||
att.key.endsWith("Screen_Shot_2022-04-21_at_2.23.26_PM.png")
|
||||
);
|
||||
|
||||
const documents = Array.from(response.documents.values());
|
||||
expect(documents.map((d) => d.text).join("")).toContain(
|
||||
attachment?.redirectUrl
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,354 @@
|
||||
import path from "path";
|
||||
import fs from "fs-extra";
|
||||
import compact from "lodash/compact";
|
||||
import escapeRegExp from "lodash/escapeRegExp";
|
||||
import mime from "mime-types";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import documentImporter from "@server/commands/documentImporter";
|
||||
import { createContext } from "@server/context";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { FileOperation, User } from "@server/models";
|
||||
import { sequelize } from "@server/storage/database";
|
||||
import ImportHelper, { FileTreeNode } from "@server/utils/ImportHelper";
|
||||
import ImportTask, { StructuredImportData } from "./ImportTask";
|
||||
|
||||
export default class ImportNotionTask extends ImportTask {
|
||||
public async parseData(
|
||||
dirPath: string,
|
||||
fileOperation: FileOperation
|
||||
): Promise<StructuredImportData> {
|
||||
const tree = await ImportHelper.toFileTree(dirPath);
|
||||
if (!tree) {
|
||||
throw new Error("Could not find valid content in zip file");
|
||||
}
|
||||
|
||||
// New Notion exports have a single folder with the name of the export, we must skip this
|
||||
// folder and go directly to the children.
|
||||
let parsed;
|
||||
if (
|
||||
tree.children.length === 1 &&
|
||||
tree.children[0].children.find((child) => child.title === "index")
|
||||
) {
|
||||
parsed = await this.parseFileTree(
|
||||
fileOperation,
|
||||
tree.children[0].children.filter((child) => child.title !== "index")
|
||||
);
|
||||
} else {
|
||||
parsed = await this.parseFileTree(fileOperation, tree.children);
|
||||
}
|
||||
|
||||
if (parsed.documents.length === 0 && parsed.collections.length === 1) {
|
||||
const collection = parsed.collections[0];
|
||||
const collectionId = uuidv4();
|
||||
if (collection.description) {
|
||||
parsed.documents.push({
|
||||
title: collection.name,
|
||||
icon: collection.icon,
|
||||
color: collection.color,
|
||||
path: "",
|
||||
text: String(collection.description),
|
||||
id: collection.id,
|
||||
externalId: collection.externalId,
|
||||
mimeType: "text/html",
|
||||
collectionId,
|
||||
});
|
||||
}
|
||||
|
||||
collection.name = "Notion";
|
||||
collection.icon = undefined;
|
||||
collection.color = undefined;
|
||||
collection.externalId = undefined;
|
||||
collection.description = undefined;
|
||||
collection.id = collectionId;
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the file structure from zipAsFileTree into documents,
|
||||
* collections, and attachments.
|
||||
*
|
||||
* @param fileOperation The file operation
|
||||
* @param tree An array of FileTreeNode representing root files in the zip
|
||||
* @returns A StructuredImportData object
|
||||
*/
|
||||
private async parseFileTree(
|
||||
fileOperation: FileOperation,
|
||||
tree: FileTreeNode[]
|
||||
): Promise<StructuredImportData> {
|
||||
const user = await User.findByPk(fileOperation.userId, {
|
||||
rejectOnEmpty: true,
|
||||
});
|
||||
|
||||
const output: StructuredImportData = {
|
||||
collections: [],
|
||||
documents: [],
|
||||
attachments: [],
|
||||
};
|
||||
|
||||
const parseNodeChildren = async (
|
||||
children: FileTreeNode[],
|
||||
collectionId: string,
|
||||
parentDocumentId?: string
|
||||
): Promise<void> => {
|
||||
await Promise.all(
|
||||
children.map(async (child) => {
|
||||
// Ignore the CSV's for databases upfront
|
||||
if (child.path.endsWith(".csv")) {
|
||||
return;
|
||||
}
|
||||
|
||||
const id = uuidv4();
|
||||
const match = child.title.match(this.NotionUUIDRegex);
|
||||
const name = child.title.replace(this.NotionUUIDRegex, "");
|
||||
const externalId = match ? match[0].trim() : undefined;
|
||||
|
||||
// If it's not a text file we're going to treat it as an attachment.
|
||||
const mimeType = mime.lookup(child.name);
|
||||
const isDocument =
|
||||
mimeType === "text/markdown" ||
|
||||
mimeType === "text/plain" ||
|
||||
mimeType === "text/html";
|
||||
|
||||
// If it's not a document and not a folder, treat it as an attachment
|
||||
if (!isDocument && mimeType) {
|
||||
output.attachments.push({
|
||||
id,
|
||||
name: child.name,
|
||||
path: child.path,
|
||||
mimeType,
|
||||
buffer: () => fs.readFile(child.path),
|
||||
externalId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
Logger.debug("task", `Processing ${name} as ${mimeType}`);
|
||||
|
||||
const { title, icon, text } = await sequelize.transaction(
|
||||
async (transaction) =>
|
||||
documentImporter({
|
||||
mimeType: mimeType || "text/markdown",
|
||||
fileName: name,
|
||||
content:
|
||||
child.children.length > 0
|
||||
? ""
|
||||
: await fs.readFile(child.path, "utf8"),
|
||||
user,
|
||||
ctx: createContext({ user, transaction }),
|
||||
})
|
||||
);
|
||||
|
||||
const existingDocumentIndex = output.documents.findIndex(
|
||||
(doc) => doc.externalId === externalId
|
||||
);
|
||||
|
||||
const existingDocument = output.documents[existingDocumentIndex];
|
||||
|
||||
// If there is an existing document with the same externalId that means
|
||||
// we've already parsed either a folder or a file referencing the same
|
||||
// document, as such we should merge.
|
||||
if (existingDocument) {
|
||||
if (existingDocument.text === "") {
|
||||
output.documents[existingDocumentIndex].text = text;
|
||||
}
|
||||
|
||||
await parseNodeChildren(
|
||||
child.children,
|
||||
collectionId,
|
||||
existingDocument.id
|
||||
);
|
||||
} else {
|
||||
output.documents.push({
|
||||
id,
|
||||
title,
|
||||
icon,
|
||||
text,
|
||||
collectionId,
|
||||
parentDocumentId,
|
||||
path: child.path,
|
||||
mimeType: mimeType || "text/markdown",
|
||||
externalId,
|
||||
});
|
||||
await parseNodeChildren(child.children, collectionId, id);
|
||||
}
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const replaceInternalLinksAndImages = (text: string) => {
|
||||
// Find if there are any images in this document
|
||||
const imagesInText = this.parseImages(text);
|
||||
|
||||
for (const image of imagesInText) {
|
||||
const name = path.basename(image.src);
|
||||
const attachment = output.attachments.find(
|
||||
(att) =>
|
||||
att.path.endsWith(image.src) ||
|
||||
encodeURI(att.path).endsWith(image.src)
|
||||
);
|
||||
|
||||
if (!attachment) {
|
||||
if (!image.src.startsWith("http")) {
|
||||
Logger.info(
|
||||
"task",
|
||||
`Could not find referenced attachment with name ${name} and src ${image.src}`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
text = text.replace(
|
||||
new RegExp(escapeRegExp(image.src), "g"),
|
||||
`<<${attachment.id}>>`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// With Notion's HTML import, images sometimes come wrapped in anchor tags
|
||||
// This isn't supported in Outline's editor, so we need to strip them.
|
||||
text = text.replace(/\[!\[([^[]+)]/g, "![]");
|
||||
|
||||
// Find if there are any links in this document pointing to other documents
|
||||
const internalLinksInText = this.parseInternalLinks(text);
|
||||
|
||||
// For each link update to the standardized format of <<documentId>>
|
||||
// instead of a relative or absolute URL within the original zip file.
|
||||
for (const link of internalLinksInText) {
|
||||
const doc = output.documents.find(
|
||||
(doc) => doc.externalId === link.externalId
|
||||
);
|
||||
|
||||
if (!doc) {
|
||||
Logger.info(
|
||||
"task",
|
||||
`Could not find referenced document with externalId ${link.externalId}`
|
||||
);
|
||||
} else {
|
||||
text = text.replace(link.href, `<<${doc.id}>>`);
|
||||
}
|
||||
}
|
||||
|
||||
return text;
|
||||
};
|
||||
|
||||
// All nodes in the root level should become collections
|
||||
for (const node of tree) {
|
||||
const match = node.title.match(this.NotionUUIDRegex);
|
||||
const name = node.title.replace(this.NotionUUIDRegex, "");
|
||||
const externalId = match ? match[0].trim() : undefined;
|
||||
const mimeType = mime.lookup(node.name);
|
||||
|
||||
const existingCollectionIndex = output.collections.findIndex(
|
||||
(collection) => collection.externalId === externalId
|
||||
);
|
||||
const existingCollection = output.collections[existingCollectionIndex];
|
||||
const collectionId = existingCollection?.id || uuidv4();
|
||||
let description;
|
||||
|
||||
// Root level docs become the descriptions of collections
|
||||
if (
|
||||
mimeType === "text/markdown" ||
|
||||
mimeType === "text/plain" ||
|
||||
mimeType === "text/html"
|
||||
) {
|
||||
const { text } = await sequelize.transaction(async (transaction) =>
|
||||
documentImporter({
|
||||
mimeType,
|
||||
fileName: name,
|
||||
content: await fs.readFile(node.path, "utf8"),
|
||||
user,
|
||||
ctx: createContext({ user, transaction }),
|
||||
})
|
||||
);
|
||||
|
||||
description = text;
|
||||
} else if (node.children.length > 0) {
|
||||
await parseNodeChildren(node.children, collectionId);
|
||||
} else {
|
||||
Logger.debug("task", `Unhandled file in zip: ${node.path}`, {
|
||||
fileOperationId: fileOperation.id,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existingCollectionIndex !== -1) {
|
||||
if (description) {
|
||||
output.collections[existingCollectionIndex].description = description;
|
||||
}
|
||||
} else {
|
||||
output.collections.push({
|
||||
id: collectionId,
|
||||
name,
|
||||
description,
|
||||
externalId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const document of output.documents) {
|
||||
document.text = replaceInternalLinksAndImages(document.text);
|
||||
}
|
||||
|
||||
for (const collection of output.collections) {
|
||||
if (typeof collection.description === "string") {
|
||||
collection.description = replaceInternalLinksAndImages(
|
||||
collection.description
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts internal links from a markdown document, taking into account the
|
||||
* externalId of the document, which is part of the link title.
|
||||
*
|
||||
* @param text The markdown text to parse
|
||||
* @returns An array of internal links
|
||||
*/
|
||||
private parseInternalLinks(
|
||||
text: string
|
||||
): { title: string; href: string; externalId: string }[] {
|
||||
return compact(
|
||||
[...text.matchAll(this.NotionLinkRegex)].map((match) => ({
|
||||
title: match[1],
|
||||
href: match[2],
|
||||
externalId: match[3],
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts images from the markdown document
|
||||
*
|
||||
* @param text The markdown text to parse
|
||||
* @returns An array of internal links
|
||||
*/
|
||||
private parseImages(text: string): { alt: string; src: string }[] {
|
||||
return compact(
|
||||
[...text.matchAll(this.ImageRegex)].map((match) => ({
|
||||
alt: match[1],
|
||||
src: match[2],
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Regex to find markdown images of all types
|
||||
*/
|
||||
private ImageRegex =
|
||||
/!\[(?<alt>[^\][]*?)]\((?<filename>[^\][]*?)(?=“|\))“?(?<title>[^\][”]+)?”?\)/g;
|
||||
|
||||
/**
|
||||
* Regex to find markdown links containing ID's that look like UUID's with the
|
||||
* "-"'s removed, Notion's externalId format.
|
||||
*/
|
||||
private NotionLinkRegex = /\[([^[]+)]\((.*?([0-9a-fA-F]{32})\..*?)\)/g;
|
||||
|
||||
/**
|
||||
* Regex to find Notion document UUID's in the title of a document.
|
||||
*/
|
||||
private NotionUUIDRegex =
|
||||
/\s([0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}|[0-9a-fA-F]{32})$/;
|
||||
}
|
||||
@@ -597,7 +597,6 @@ router.post(
|
||||
createdById: user.id,
|
||||
},
|
||||
transaction,
|
||||
hooks: false,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import queryString from "query-string";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomElement } from "@shared/random";
|
||||
import { NotificationEventType } from "@shared/types";
|
||||
import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper";
|
||||
import {
|
||||
buildCollection,
|
||||
buildDocument,
|
||||
@@ -698,40 +697,3 @@ describe("#notifications.update_all", () => {
|
||||
expect(body.data.total).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#notifications.unsubscribe", () => {
|
||||
it("should allow unsubscribe with valid token", async () => {
|
||||
const user = await buildUser();
|
||||
const token = NotificationSettingsHelper.unsubscribeToken(
|
||||
user.id,
|
||||
NotificationEventType.UpdateDocument
|
||||
);
|
||||
|
||||
const res = await server.get(
|
||||
`/api/notifications.unsubscribe?userId=${user.id}&token=${token}&eventType=documents.update&follow=true`,
|
||||
{
|
||||
redirect: "manual",
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(302);
|
||||
expect(res.headers.get("location")).toContain(
|
||||
"/settings/notifications?success"
|
||||
);
|
||||
|
||||
const events = (await user.reload()).notificationSettings;
|
||||
expect(events).not.toContain("documents.update");
|
||||
});
|
||||
|
||||
it("should not allow unsubscribe with invalid token", async () => {
|
||||
const user = await buildUser();
|
||||
|
||||
const res = await server.get(
|
||||
`/api/notifications.unsubscribe?userId=${user.id}&token=invalid-token&eventType=documents.update&follow=true`,
|
||||
{
|
||||
redirect: "manual",
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(302);
|
||||
expect(res.headers.get("location")).toContain("?notice=invalid-auth");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,7 +53,7 @@ const handleUnsubscribe = async (
|
||||
});
|
||||
|
||||
user.setNotificationEventType(eventType, false);
|
||||
await user.save({ transaction });
|
||||
await user.save();
|
||||
ctx.redirect(`${user.team.url}/settings/notifications?success`);
|
||||
};
|
||||
|
||||
|
||||
Vendored
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -18,14 +18,11 @@ export class CacheHelper {
|
||||
* @param key Cache key
|
||||
* @param callback Callback to get the data if not found in cache
|
||||
* @param expiry Cache data expiry in seconds
|
||||
* @param lockTimeout Lock timeout in milliseconds
|
||||
* @returns The data from cache or the result of the callback
|
||||
*/
|
||||
public static async getDataOrSet<T>(
|
||||
key: string,
|
||||
callback: () => Promise<T | undefined>,
|
||||
expiry: number,
|
||||
lockTimeout: number = MutexLock.defaultLockTimeout
|
||||
expiry?: number
|
||||
): Promise<T | undefined> {
|
||||
let cache = await this.getData<T>(key);
|
||||
|
||||
@@ -38,7 +35,10 @@ export class CacheHelper {
|
||||
const lockKey = `lock:${key}`;
|
||||
try {
|
||||
try {
|
||||
lock = await MutexLock.lock.acquire([lockKey], lockTimeout);
|
||||
lock = await MutexLock.lock.acquire(
|
||||
[lockKey],
|
||||
MutexLock.defaultLockTimeout
|
||||
);
|
||||
} catch (err) {
|
||||
Logger.error(`Could not acquire lock for ${key}`, err);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { generateAvatarUrl } from "./avatars";
|
||||
|
||||
it("should return clearbit url if available", async () => {
|
||||
const url = await generateAvatarUrl({
|
||||
id: "google",
|
||||
domain: "google.com",
|
||||
});
|
||||
expect(url).toBe("https://logo.clearbit.com/google.com");
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import crypto from "crypto";
|
||||
import fetch from "./fetch";
|
||||
|
||||
export async function generateAvatarUrl({
|
||||
id,
|
||||
domain,
|
||||
}: {
|
||||
id: string;
|
||||
domain?: string;
|
||||
}) {
|
||||
// attempt to get logo from Clearbit API. If one doesn't exist then
|
||||
// fall back to using tiley to generate a placeholder logo
|
||||
const hash = crypto.createHash("sha256");
|
||||
hash.update(id);
|
||||
let cbResponse, cbUrl;
|
||||
|
||||
if (domain) {
|
||||
cbUrl = `https://logo.clearbit.com/${domain}`;
|
||||
|
||||
try {
|
||||
cbResponse = await fetch(cbUrl);
|
||||
} catch (err) {
|
||||
// okay
|
||||
}
|
||||
}
|
||||
|
||||
return cbUrl && cbResponse && cbResponse.status === 200 ? cbUrl : null;
|
||||
}
|
||||
@@ -28,10 +28,9 @@ export function getCellAttrs(dom: HTMLElement | string): Attrs {
|
||||
const widthAttr = dom.getAttribute("data-colwidth");
|
||||
const widths =
|
||||
widthAttr && /^\d+(,\d+)*$/.test(widthAttr)
|
||||
? widthAttr.split(",").map(Number)
|
||||
? widthAttr.split(",").map((s) => Number(s))
|
||||
: null;
|
||||
const colspan = Number(dom.getAttribute("colspan") || 1);
|
||||
|
||||
return {
|
||||
colspan,
|
||||
rowspan: Number(dom.getAttribute("rowspan") || 1),
|
||||
@@ -64,11 +63,10 @@ export function setCellAttrs(node: Node): Attrs {
|
||||
}
|
||||
if (node.attrs.colwidth) {
|
||||
if (isBrowser) {
|
||||
attrs["data-colwidth"] = node.attrs.colwidth.map(parseInt).join(",");
|
||||
attrs["data-colwidth"] = node.attrs.colwidth.join(",");
|
||||
} else {
|
||||
attrs.style =
|
||||
(attrs.style ?? "") +
|
||||
`min-width: ${parseInt(node.attrs.colwidth[0])}px;`;
|
||||
(attrs.style ?? "") + `min-width: ${node.attrs.colwidth}px;`;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ export default class Code extends Mark {
|
||||
excludes: "mention placeholder highlight",
|
||||
parseDOM: [{ tag: "code", preserveWhitespace: true }],
|
||||
toDOM: () => ["code", { class: "inline", spellCheck: "false" }],
|
||||
code: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -98,6 +98,8 @@ export default class TableCell extends Node {
|
||||
}
|
||||
}
|
||||
|
||||
// For all other cases, preserve the slice as is to maintain table structure
|
||||
// including column widths and other attributes
|
||||
return slice;
|
||||
},
|
||||
handleDOMEvents: {
|
||||
|
||||
@@ -903,6 +903,9 @@
|
||||
"{{ count }} document imported_plural": "{{ count }} documents imported",
|
||||
"You can import a zip file that was previously exported from an Outline installation – collections, documents, and images will be imported. In Outline, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.": "You can import a zip file that was previously exported from an Outline installation – collections, documents, and images will be imported. In Outline, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.",
|
||||
"Drag and drop the zip file from the Markdown export option in {{appName}}, or click to upload": "Drag and drop the zip file from the Markdown export option in {{appName}}, or click to upload",
|
||||
"Where do I find the file?": "Where do I find the file?",
|
||||
"In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability.": "In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability.",
|
||||
"Drag and drop the zip file from Notion's HTML export option, or click to upload": "Drag and drop the zip file from Notion's HTML export option, or click to upload",
|
||||
"Last active": "Last active",
|
||||
"Guest": "Guest",
|
||||
"Shared by": "Shared by",
|
||||
|
||||
Reference in New Issue
Block a user