mirror of
https://github.com/outline/outline.git
synced 2026-06-13 11:25:03 +03:00
Compare commits
23 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ffcd187a16 | |||
| 01a5ff031a | |||
| 5659aeb360 | |||
| d300e34447 | |||
| a4040a93a2 | |||
| c769432993 | |||
| 270bb85417 | |||
| fe8e50da92 | |||
| 31d1f566bc | |||
| f9476770ce | |||
| 2e018e74b8 | |||
| a11ab56117 | |||
| 66e4ec32ed | |||
| bde9d5fbf4 | |||
| 70bb878a8c | |||
| 4237377d47 | |||
| a30f6b717b | |||
| 1edc23c5ae | |||
| ff6ec3a5b8 | |||
| 52c2729490 | |||
| 82f4281a02 | |||
| 12b6e30e3a | |||
| 567ca7e3f1 |
@@ -5,7 +5,6 @@ import { Redirect } from "react-router-dom";
|
||||
import useCurrentUser from "~/hooks/useCurrentUser";
|
||||
import useStores from "~/hooks/useStores";
|
||||
import { changeLanguage } from "~/utils/language";
|
||||
import { logoutPath } from "~/utils/routeHelpers";
|
||||
import LoadingIndicator from "./LoadingIndicator";
|
||||
|
||||
type Props = {
|
||||
@@ -33,7 +32,7 @@ const Authenticated = ({ children }: Props) => {
|
||||
}
|
||||
|
||||
void auth.logout(true);
|
||||
return <Redirect to={logoutPath()} />;
|
||||
return <Redirect to="/" />;
|
||||
};
|
||||
|
||||
export default observer(Authenticated);
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
import { AnimatePresence } from "framer-motion";
|
||||
import { observer } from "mobx-react";
|
||||
import * as React from "react";
|
||||
import { Switch, Route, useLocation, matchPath } from "react-router-dom";
|
||||
import {
|
||||
Switch,
|
||||
Route,
|
||||
useLocation,
|
||||
matchPath,
|
||||
Redirect,
|
||||
} from "react-router-dom";
|
||||
import { TeamPreference } from "@shared/types";
|
||||
import ErrorSuspended from "~/scenes/Errors/ErrorSuspended";
|
||||
import Layout from "~/components/Layout";
|
||||
@@ -10,6 +16,7 @@ import Sidebar from "~/components/Sidebar";
|
||||
import SidebarRight from "~/components/Sidebar/Right";
|
||||
import SettingsSidebar from "~/components/Sidebar/Settings";
|
||||
import useCurrentTeam from "~/hooks/useCurrentTeam";
|
||||
import { usePostLoginPath } from "~/hooks/useLastVisitedPath";
|
||||
import usePolicy from "~/hooks/usePolicy";
|
||||
import useStores from "~/hooks/useStores";
|
||||
import history from "~/utils/history";
|
||||
@@ -48,6 +55,7 @@ const AuthenticatedLayout: React.FC = ({ children }: Props) => {
|
||||
const can = usePolicy(ui.activeDocumentId);
|
||||
const canCollection = usePolicy(ui.activeCollectionId);
|
||||
const team = useCurrentTeam();
|
||||
const [spendPostLoginPath] = usePostLoginPath();
|
||||
|
||||
const goToSearch = (ev: KeyboardEvent) => {
|
||||
if (!ev.metaKey && !ev.ctrlKey) {
|
||||
@@ -72,6 +80,11 @@ const AuthenticatedLayout: React.FC = ({ children }: Props) => {
|
||||
return <ErrorSuspended />;
|
||||
}
|
||||
|
||||
const postLoginPath = spendPostLoginPath();
|
||||
if (postLoginPath) {
|
||||
return <Redirect to={postLoginPath} />;
|
||||
}
|
||||
|
||||
const sidebar = (
|
||||
<Fade>
|
||||
<Switch>
|
||||
|
||||
@@ -45,6 +45,10 @@ export const NativeInput = styled.input<{
|
||||
${ellipsis()}
|
||||
${undraggableOnDesktop()}
|
||||
|
||||
&[readOnly] {
|
||||
color: ${s("textSecondary")};
|
||||
}
|
||||
|
||||
&:disabled,
|
||||
&::placeholder {
|
||||
color: ${s("placeholder")};
|
||||
@@ -126,13 +130,14 @@ export interface Props
|
||||
React.InputHTMLAttributes<HTMLInputElement | HTMLTextAreaElement>,
|
||||
"prefix"
|
||||
> {
|
||||
type?: "text" | "email" | "checkbox" | "search" | "textarea";
|
||||
type?: "text" | "email" | "checkbox" | "search" | "textarea" | "password";
|
||||
labelHidden?: boolean;
|
||||
label?: string;
|
||||
flex?: boolean;
|
||||
short?: boolean;
|
||||
margin?: string | number;
|
||||
error?: string;
|
||||
rows?: number;
|
||||
/** Optional component that appears inside the input before the textarea and any icon */
|
||||
prefix?: React.ReactNode;
|
||||
/** Optional icon that appears inside the input before the textarea */
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
import { observer } from "mobx-react";
|
||||
import * as React from "react";
|
||||
import styled from "styled-components";
|
||||
import { s } from "@shared/styles";
|
||||
import Flex from "~/components/Flex";
|
||||
|
||||
type Props = {
|
||||
children?: React.ReactNode;
|
||||
label: React.ReactNode | string;
|
||||
};
|
||||
|
||||
const Labeled: React.FC<Props> = ({ label, children, ...props }: Props) => (
|
||||
<Flex column {...props}>
|
||||
<Label>{label}</Label>
|
||||
{children}
|
||||
</Flex>
|
||||
);
|
||||
|
||||
export const Label = styled(Flex)`
|
||||
font-weight: 500;
|
||||
padding-bottom: 4px;
|
||||
display: inline-block;
|
||||
color: ${s("text")};
|
||||
`;
|
||||
|
||||
export default observer(Labeled);
|
||||
+1
-8
@@ -2,7 +2,7 @@ import { observer } from "mobx-react";
|
||||
import { HomeIcon } from "outline-icons";
|
||||
import * as React from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Switch, Route, Redirect } from "react-router-dom";
|
||||
import { Switch, Route } from "react-router-dom";
|
||||
import styled from "styled-components";
|
||||
import { s } from "@shared/styles";
|
||||
import { Action } from "~/components/Actions";
|
||||
@@ -18,7 +18,6 @@ import Tab from "~/components/Tab";
|
||||
import Tabs from "~/components/Tabs";
|
||||
import useCurrentTeam from "~/hooks/useCurrentTeam";
|
||||
import useCurrentUser from "~/hooks/useCurrentUser";
|
||||
import { usePostLoginPath } from "~/hooks/useLastVisitedPath";
|
||||
import { usePinnedDocuments } from "~/hooks/usePinnedDocuments";
|
||||
import usePolicy from "~/hooks/usePolicy";
|
||||
import useStores from "~/hooks/useStores";
|
||||
@@ -29,16 +28,10 @@ function Home() {
|
||||
const team = useCurrentTeam();
|
||||
const user = useCurrentUser();
|
||||
const { t } = useTranslation();
|
||||
const [spendPostLoginPath] = usePostLoginPath();
|
||||
const userId = user?.id;
|
||||
const { pins, count } = usePinnedDocuments("home");
|
||||
const can = usePolicy(team);
|
||||
|
||||
const postLoginPath = spendPostLoginPath();
|
||||
if (postLoginPath) {
|
||||
return <Redirect to={postLoginPath} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<Scene
|
||||
icon={<HomeIcon />}
|
||||
|
||||
@@ -9,7 +9,7 @@ import { s } from "@shared/styles";
|
||||
import { AttachmentPreset } from "@shared/types";
|
||||
import { AttachmentValidation } from "@shared/validations";
|
||||
import RootStore from "~/stores/RootStore";
|
||||
import Button from "~/components/Button";
|
||||
import ButtonLarge from "~/components/ButtonLarge";
|
||||
import Flex from "~/components/Flex";
|
||||
import LoadingIndicator from "~/components/LoadingIndicator";
|
||||
import Modal from "~/components/Modal";
|
||||
@@ -95,8 +95,14 @@ class ImageUpload extends React.Component<RootStore & Props> {
|
||||
|
||||
renderCropping() {
|
||||
const { ui, submitText } = this.props;
|
||||
|
||||
return (
|
||||
<Modal isOpen onRequestClose={this.handleClose} title="">
|
||||
<Modal
|
||||
onRequestClose={this.handleClose}
|
||||
fullscreen={false}
|
||||
title={<> </>}
|
||||
isOpen
|
||||
>
|
||||
<Flex auto column align="center" justify="center">
|
||||
{this.isUploading && <LoadingIndicator />}
|
||||
<AvatarEditorContainer>
|
||||
@@ -122,9 +128,14 @@ class ImageUpload extends React.Component<RootStore & Props> {
|
||||
defaultValue="1"
|
||||
onChange={this.handleZoom}
|
||||
/>
|
||||
<CropButton onClick={this.handleCrop} disabled={this.isUploading}>
|
||||
<br />
|
||||
<ButtonLarge
|
||||
fullwidth
|
||||
onClick={this.handleCrop}
|
||||
disabled={this.isUploading}
|
||||
>
|
||||
{this.isUploading ? "Uploading…" : submitText}
|
||||
</CropButton>
|
||||
</ButtonLarge>
|
||||
</Flex>
|
||||
</Modal>
|
||||
);
|
||||
@@ -180,8 +191,4 @@ const RangeInput = styled.input`
|
||||
}
|
||||
`;
|
||||
|
||||
const CropButton = styled(Button)`
|
||||
width: 300px;
|
||||
`;
|
||||
|
||||
export default withStores(ImageUpload);
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
import * as React from "react";
|
||||
import { Trans, useTranslation } from "react-i18next";
|
||||
import { FileOperationFormat } from "@shared/types";
|
||||
import useStores from "~/hooks/useStores";
|
||||
import DropToImport from "./DropToImport";
|
||||
import HelpDisclosure from "./HelpDisclosure";
|
||||
|
||||
function ImportNotionDialog() {
|
||||
const { t } = useTranslation();
|
||||
const { dialogs } = useStores();
|
||||
|
||||
return (
|
||||
<>
|
||||
<HelpDisclosure title={<Trans>Where do I find the file?</Trans>}>
|
||||
<Trans
|
||||
defaults="In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability."
|
||||
components={{
|
||||
em: <strong />,
|
||||
}}
|
||||
/>
|
||||
</HelpDisclosure>
|
||||
<DropToImport
|
||||
onSubmit={dialogs.closeAllModals}
|
||||
format={FileOperationFormat.Notion}
|
||||
>
|
||||
<>
|
||||
{t(
|
||||
`Drag and drop the zip file from Notion's HTML export option, or click to upload`
|
||||
)}
|
||||
</>
|
||||
</DropToImport>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default ImportNotionDialog;
|
||||
@@ -306,7 +306,7 @@ export default class AuthStore extends Store<Team> {
|
||||
// if this logout was forced from an authenticated route then
|
||||
// save the current path so we can go back there once signed in
|
||||
if (savePath) {
|
||||
setPostLoginPath(window.location.pathname);
|
||||
setPostLoginPath(window.location.pathname + window.location.search);
|
||||
}
|
||||
|
||||
if (tryRevokingToken) {
|
||||
|
||||
+5
-3
@@ -49,8 +49,10 @@ export function redirectTo(url: string) {
|
||||
/**
|
||||
* Check if the path is a valid path for redirect after login.
|
||||
*
|
||||
* @param path
|
||||
* @param input A path potentially including query string
|
||||
* @returns boolean indicating if the path is a valid redirect
|
||||
*/
|
||||
export const isAllowedLoginRedirect = (path: string) =>
|
||||
!["/", "/create", "/home", "/logout", "/auth/"].includes(path);
|
||||
export const isAllowedLoginRedirect = (input: string) => {
|
||||
const path = input.split("?")[0];
|
||||
return !["/", "/create", "/home", "/logout", "/auth/"].includes(path);
|
||||
};
|
||||
|
||||
+14
-14
@@ -48,11 +48,11 @@
|
||||
"> 0.25%, not dead"
|
||||
],
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.774.0",
|
||||
"@aws-sdk/lib-storage": "3.774.0",
|
||||
"@aws-sdk/s3-presigned-post": "3.774.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.774.0",
|
||||
"@aws-sdk/signature-v4-crt": "^3.774.0",
|
||||
"@aws-sdk/client-s3": "3.777.0",
|
||||
"@aws-sdk/lib-storage": "3.777.0",
|
||||
"@aws-sdk/s3-presigned-post": "3.777.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.777.0",
|
||||
"@aws-sdk/signature-v4-crt": "^3.775.0",
|
||||
"@babel/core": "^7.26.10",
|
||||
"@babel/plugin-proposal-decorators": "^7.25.9",
|
||||
"@babel/plugin-transform-class-properties": "^7.25.9",
|
||||
@@ -89,7 +89,7 @@
|
||||
"@sentry/node": "^7.120.3",
|
||||
"@sentry/react": "^7.120.3",
|
||||
"@tanstack/react-table": "^8.20.6",
|
||||
"@tanstack/react-virtual": "^3.11.3",
|
||||
"@tanstack/react-virtual": "^3.13.6",
|
||||
"@tippyjs/react": "^4.2.6",
|
||||
"@types/form-data": "^2.5.2",
|
||||
"@types/mailparser": "^3.4.5",
|
||||
@@ -153,7 +153,7 @@
|
||||
"koa-useragent": "^4.1.0",
|
||||
"lodash": "^4.17.21",
|
||||
"mailparser": "^3.7.2",
|
||||
"mammoth": "^1.8.0",
|
||||
"mammoth": "^1.9.0",
|
||||
"markdown-it": "^13.0.2",
|
||||
"markdown-it-container": "^3.0.0",
|
||||
"markdown-it-emoji": "^2.0.0",
|
||||
@@ -183,10 +183,10 @@
|
||||
"prosemirror-dropcursor": "^1.8.1",
|
||||
"prosemirror-gapcursor": "^1.3.2",
|
||||
"prosemirror-history": "^1.4.1",
|
||||
"prosemirror-inputrules": "^1.4.0",
|
||||
"prosemirror-inputrules": "^1.5.0",
|
||||
"prosemirror-keymap": "^1.2.2",
|
||||
"prosemirror-markdown": "^1.13.1",
|
||||
"prosemirror-model": "^1.24.0",
|
||||
"prosemirror-markdown": "^1.13.2",
|
||||
"prosemirror-model": "^1.25.0",
|
||||
"prosemirror-schema-list": "^1.4.1",
|
||||
"prosemirror-state": "^1.4.3",
|
||||
"prosemirror-tables": "^1.6.4",
|
||||
@@ -232,7 +232,7 @@
|
||||
"socket.io": "^4.8.1",
|
||||
"socket.io-client": "^4.8.1",
|
||||
"socket.io-redis": "^6.1.1",
|
||||
"sonner": "^1.7.1",
|
||||
"sonner": "^1.7.4",
|
||||
"stoppable": "^1.1.0",
|
||||
"string-replace-to-array": "^2.1.1",
|
||||
"styled-components": "^5.3.11",
|
||||
@@ -248,7 +248,7 @@
|
||||
"uuid": "^8.3.2",
|
||||
"validator": "13.12.0",
|
||||
"vaul": "^1.1.2",
|
||||
"vite": "^5.4.15",
|
||||
"vite": "^5.4.17",
|
||||
"vite-plugin-pwa": "^0.20.3",
|
||||
"winston": "^3.17.0",
|
||||
"ws": "^7.5.10",
|
||||
@@ -263,7 +263,7 @@
|
||||
"@babel/cli": "^7.27.0",
|
||||
"@babel/preset-typescript": "^7.27.0",
|
||||
"@faker-js/faker": "^8.4.1",
|
||||
"@relative-ci/agent": "^4.2.14",
|
||||
"@relative-ci/agent": "^4.3.0",
|
||||
"@testing-library/react": "^12.0.0",
|
||||
"@types/addressparser": "^1.0.3",
|
||||
"@types/body-scroll-lock": "^3.1.2",
|
||||
@@ -296,7 +296,7 @@
|
||||
"@types/markdown-it-emoji": "^2.0.4",
|
||||
"@types/mime-types": "^2.1.4",
|
||||
"@types/natural-sort": "^0.0.24",
|
||||
"@types/node": "20.17.27",
|
||||
"@types/node": "20.17.30",
|
||||
"@types/node-fetch": "^2.6.9",
|
||||
"@types/nodemailer": "^6.4.17",
|
||||
"@types/passport-oauth2": "^1.4.17",
|
||||
|
||||
@@ -28,7 +28,7 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
|
||||
{
|
||||
clientID: env.AZURE_CLIENT_ID,
|
||||
clientSecret: env.AZURE_CLIENT_SECRET,
|
||||
callbackURL: `${env.URL}/auth/azure.callback`,
|
||||
callbackURL: ,
|
||||
useCommonEndpoint: env.AZURE_TENANT_ID ? false : true,
|
||||
tenant: env.AZURE_TENANT_ID ? env.AZURE_TENANT_ID : undefined,
|
||||
passReqToCallback: true,
|
||||
@@ -57,12 +57,12 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
|
||||
const [profileResponse, organizationResponse] = await Promise.all([
|
||||
// Load the users profile from the Microsoft Graph API
|
||||
// https://docs.microsoft.com/en-us/graph/api/resources/users?view=graph-rest-1.0
|
||||
request("GET", `https://graph.microsoft.com/v1.0/me`, accessToken),
|
||||
request("GET", , accessToken),
|
||||
// Load the organization profile from the Microsoft Graph API
|
||||
// https://docs.microsoft.com/en-us/graph/api/organization-get?view=graph-rest-1.0
|
||||
request(
|
||||
"GET",
|
||||
`https://graph.microsoft.com/v1.0/organization`,
|
||||
,
|
||||
accessToken
|
||||
),
|
||||
]);
|
||||
@@ -73,14 +73,6 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
|
||||
);
|
||||
}
|
||||
|
||||
if (!organizationResponse?.value?.length) {
|
||||
throw MicrosoftGraphError(
|
||||
`Unable to load organization info from Microsoft Graph API: ${organizationResponse.error?.message}`
|
||||
);
|
||||
}
|
||||
|
||||
const organization = organizationResponse.value[0];
|
||||
|
||||
// Note: userPrincipalName is last here for backwards compatibility with
|
||||
// previous versions of Outline that did not include it.
|
||||
const email =
|
||||
@@ -100,7 +92,17 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
|
||||
const domain = parseEmail(email).domain;
|
||||
const subdomain = slugifyDomain(domain);
|
||||
|
||||
const teamName = organization.displayName;
|
||||
// Default team name to use if organization display name is not available
|
||||
let teamName = domain;
|
||||
|
||||
// Try to get the organization display name if available
|
||||
if (organizationResponse?.value?.length) {
|
||||
const organization = organizationResponse.value[0];
|
||||
if (organization.displayName) {
|
||||
teamName = organization.displayName;
|
||||
}
|
||||
}
|
||||
|
||||
const result = await accountProvisioner({
|
||||
ip: ctx.ip,
|
||||
team: {
|
||||
@@ -137,7 +139,7 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
|
||||
config.id,
|
||||
passport.authenticate(config.id, { prompt: "select_account" })
|
||||
);
|
||||
router.get(`${config.id}.callback`, passportMiddleware(config.id));
|
||||
router.get(, passportMiddleware(config.id));
|
||||
}
|
||||
|
||||
export default router;
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { APIResponseError, APIErrorCode } from "@notionhq/client";
|
||||
import { ImportTaskInput, ImportTaskOutput } from "@shared/schema";
|
||||
import { IntegrationService, ProsemirrorDoc } from "@shared/types";
|
||||
import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { Integration } from "@server/models";
|
||||
import ImportTask from "@server/models/ImportTask";
|
||||
import APIImportTask, {
|
||||
@@ -39,7 +41,10 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
importTask.input.map(async (item) => this.processPage({ item, client }))
|
||||
);
|
||||
|
||||
const taskOutput: ImportTaskOutput = parsedPages.map((parsedPage) => ({
|
||||
// Filter out any null results (from pages/databases that couldn't be accessed)
|
||||
const validParsedPages = parsedPages.filter(Boolean) as ParsePageOutput[];
|
||||
|
||||
const taskOutput: ImportTaskOutput = validParsedPages.map((parsedPage) => ({
|
||||
externalId: parsedPage.externalId,
|
||||
title: parsedPage.title,
|
||||
emoji: parsedPage.emoji,
|
||||
@@ -50,7 +55,7 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
}));
|
||||
|
||||
const childTasksInput: ImportTaskInput<IntegrationService.Notion> =
|
||||
parsedPages.flatMap((parsedPage) =>
|
||||
validParsedPages.flatMap((parsedPage) =>
|
||||
parsedPage.children.map((childPage) => ({
|
||||
type: childPage.type,
|
||||
externalId: childPage.externalId,
|
||||
@@ -88,36 +93,55 @@ export default class NotionAPIImportTask extends APIImportTask<IntegrationServic
|
||||
}: {
|
||||
item: ImportTaskInput<IntegrationService.Notion>[number];
|
||||
client: NotionClient;
|
||||
}): Promise<ParsePageOutput> {
|
||||
}): Promise<ParsePageOutput | null> {
|
||||
const collectionExternalId = item.collectionExternalId ?? item.externalId;
|
||||
|
||||
// Convert Notion database to an empty page with "pages in database" as its children.
|
||||
if (item.type === PageType.Database) {
|
||||
const { pages, ...databaseInfo } = await client.fetchDatabase(
|
||||
item.externalId
|
||||
);
|
||||
try {
|
||||
// Convert Notion database to an empty page with "pages in database" as its children.
|
||||
if (item.type === PageType.Database) {
|
||||
const { pages, ...databaseInfo } = await client.fetchDatabase(
|
||||
item.externalId
|
||||
);
|
||||
|
||||
return {
|
||||
...databaseInfo,
|
||||
externalId: item.externalId,
|
||||
content: ProsemirrorHelper.getEmptyDocument() as ProsemirrorDoc,
|
||||
collectionExternalId,
|
||||
children: pages.map((page) => ({
|
||||
type: page.type,
|
||||
externalId: page.id,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const { blocks, ...pageInfo } = await client.fetchPage(item.externalId);
|
||||
|
||||
return {
|
||||
...databaseInfo,
|
||||
...pageInfo,
|
||||
externalId: item.externalId,
|
||||
content: ProsemirrorHelper.getEmptyDocument() as ProsemirrorDoc,
|
||||
content: NotionConverter.page({ children: blocks } as NotionPage),
|
||||
collectionExternalId,
|
||||
children: pages.map((page) => ({
|
||||
type: page.type,
|
||||
externalId: page.id,
|
||||
})),
|
||||
children: this.parseChildPages(blocks),
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof APIResponseError) {
|
||||
// Skip this page/database if it's not found or not accessible
|
||||
if (
|
||||
error.code === APIErrorCode.ObjectNotFound ||
|
||||
error.code === APIErrorCode.Unauthorized
|
||||
) {
|
||||
Logger.warn(
|
||||
`Skipping Notion ${
|
||||
item.type === PageType.Database ? "database" : "page"
|
||||
} ${item.externalId} - Error code: ${error.code} - ${error.message}`
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
// Re-throw other errors to be handled by the parent try/catch
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { blocks, ...pageInfo } = await client.fetchPage(item.externalId);
|
||||
|
||||
return {
|
||||
...pageInfo,
|
||||
externalId: item.externalId,
|
||||
content: NotionConverter.page({ children: blocks } as NotionPage),
|
||||
collectionExternalId,
|
||||
children: this.parseChildPages(blocks),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { Server } from "@hocuspocus/server";
|
||||
import WebSocket from "ws";
|
||||
import EDITOR_VERSION from "@shared/editor/version";
|
||||
import { sleep } from "@server/utils/timers";
|
||||
import { ConnectionLimitExtension } from "./ConnectionLimitExtension";
|
||||
import { EditorVersionExtension } from "./EditorVersionExtension";
|
||||
|
||||
jest.mock("@server/env", () => ({
|
||||
COLLABORATION_MAX_CLIENTS_PER_DOCUMENT: 2,
|
||||
}));
|
||||
|
||||
describe("ConnectionLimitExtension", () => {
|
||||
let server: typeof Server;
|
||||
let extension: ConnectionLimitExtension;
|
||||
const port = 12345;
|
||||
const url = `ws://localhost:${port}`;
|
||||
const documentName = "test";
|
||||
|
||||
beforeEach(async () => {
|
||||
extension = new ConnectionLimitExtension();
|
||||
server = Server.configure({
|
||||
port,
|
||||
extensions: [extension, new EditorVersionExtension()],
|
||||
});
|
||||
await server.listen();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await server.destroy();
|
||||
});
|
||||
|
||||
const getConnections = () =>
|
||||
extension.connectionsByDocument.get(documentName)?.size ?? 0;
|
||||
|
||||
const createWebSocket = (editorVersion = EDITOR_VERSION) =>
|
||||
new Promise<WebSocket>((resolve, reject) => {
|
||||
const ws = new WebSocket(
|
||||
`${url}/${documentName}?editorVersion=${editorVersion}`
|
||||
);
|
||||
ws.on("open", () => resolve(ws));
|
||||
ws.on("error", reject);
|
||||
});
|
||||
|
||||
it("should allow connections within limit", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
expect(ws1.readyState).toBe(WebSocket.OPEN);
|
||||
expect(ws2.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should close connections exceeding limit", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
const ws3 = await createWebSocket();
|
||||
await sleep(250);
|
||||
|
||||
expect(ws3.readyState).toBe(WebSocket.CLOSED);
|
||||
expect(ws2.readyState).toBe(WebSocket.OPEN);
|
||||
expect(ws1.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should handle connections closed by other extensions", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
|
||||
// Create a connection that will be closed by the EditorVersionExtension
|
||||
const ws2 = await createWebSocket("1.0.0");
|
||||
|
||||
ws1.close();
|
||||
ws2.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
|
||||
it("should allow new connection after disconnect", async () => {
|
||||
const ws1 = await createWebSocket();
|
||||
const ws2 = await createWebSocket();
|
||||
|
||||
ws1.close();
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(1);
|
||||
|
||||
const ws3 = await createWebSocket();
|
||||
expect(ws3.readyState).toBe(WebSocket.OPEN);
|
||||
expect(getConnections()).toBe(2);
|
||||
|
||||
ws2.close();
|
||||
ws3.close();
|
||||
|
||||
await sleep(250);
|
||||
expect(getConnections()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,10 @@
|
||||
import {
|
||||
Extension,
|
||||
connectedPayload,
|
||||
onConnectPayload,
|
||||
onDisconnectPayload,
|
||||
} from "@hocuspocus/server";
|
||||
import pluralize from "pluralize";
|
||||
import { TooManyConnections } from "@shared/collaboration/CloseEvents";
|
||||
import env from "@server/env";
|
||||
import Logger from "@server/logging/Logger";
|
||||
@@ -14,7 +16,7 @@ export class ConnectionLimitExtension implements Extension {
|
||||
/**
|
||||
* Map of documentId -> connection count
|
||||
*/
|
||||
connectionsByDocument: Map<string, Set<string>> = new Map();
|
||||
public connectionsByDocument: Map<string, Set<string>> = new Map();
|
||||
|
||||
/**
|
||||
* On disconnect hook
|
||||
@@ -34,23 +36,30 @@ export class ConnectionLimitExtension implements Extension {
|
||||
}
|
||||
}
|
||||
|
||||
const connectionCount = connections?.size ?? 0;
|
||||
Logger.debug(
|
||||
"multiplayer",
|
||||
`${connections?.size} connections to "${documentName}"`
|
||||
`${connectionCount} ${pluralize(
|
||||
"connection",
|
||||
connectionCount
|
||||
)} to "${documentName}"`
|
||||
);
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
/**
|
||||
* On connect hook
|
||||
* onConnect hook is called when a new connection has been established.
|
||||
* This is where we can check if the document has reached the maximum number of
|
||||
* connections and reject the connection if it has.
|
||||
*
|
||||
* @param data The connect payload
|
||||
* @returns Promise, resolving will allow the connection, rejecting will drop it
|
||||
* @param data The onConnect payload
|
||||
* @returns Promise, resolving will allow the connection, rejecting will drop.
|
||||
*/
|
||||
onConnect({ documentName, socketId }: withContext<onConnectPayload>) {
|
||||
onConnect({ documentName }: withContext<onConnectPayload>) {
|
||||
const connections =
|
||||
this.connectionsByDocument.get(documentName) || new Set();
|
||||
|
||||
if (connections?.size >= env.COLLABORATION_MAX_CLIENTS_PER_DOCUMENT) {
|
||||
Logger.info(
|
||||
"multiplayer",
|
||||
@@ -61,12 +70,30 @@ export class ConnectionLimitExtension implements Extension {
|
||||
return Promise.reject(TooManyConnections);
|
||||
}
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
/**
|
||||
* Connected hook is called after a new connection has been established.
|
||||
* We can safely update the connection count for the document.
|
||||
*
|
||||
* @param data The onConnect payload
|
||||
* @returns Promise
|
||||
*/
|
||||
connected({ documentName, socketId }: withContext<connectedPayload>) {
|
||||
const connections =
|
||||
this.connectionsByDocument.get(documentName) || new Set();
|
||||
|
||||
connections.add(socketId);
|
||||
this.connectionsByDocument.set(documentName, connections);
|
||||
const connectionCount = connections.size ?? 0;
|
||||
|
||||
Logger.debug(
|
||||
"multiplayer",
|
||||
`${connections.size} connections to "${documentName}"`
|
||||
`${connectionCount} ${pluralize(
|
||||
"connection",
|
||||
connectionCount
|
||||
)} to "${documentName}"`
|
||||
);
|
||||
|
||||
return Promise.resolve();
|
||||
|
||||
@@ -3,7 +3,6 @@ import slugify from "slugify";
|
||||
import { RESERVED_SUBDOMAINS } from "@shared/utils/domains";
|
||||
import { traceFunction } from "@server/logging/tracing";
|
||||
import { Team, Event } from "@server/models";
|
||||
import { generateAvatarUrl } from "@server/utils/avatars";
|
||||
|
||||
type Props = {
|
||||
/** The displayed name of the team */
|
||||
@@ -29,20 +28,14 @@ type Props = {
|
||||
|
||||
async function teamCreator({
|
||||
name,
|
||||
domain,
|
||||
subdomain,
|
||||
avatarUrl,
|
||||
authenticationProviders,
|
||||
ip,
|
||||
transaction,
|
||||
}: Props): Promise<Team> {
|
||||
// If the service did not provide a logo/avatar then we attempt to generate
|
||||
// one via ClearBit, or fallback to colored initials in worst case scenario
|
||||
if (!avatarUrl || !avatarUrl.startsWith("http")) {
|
||||
avatarUrl = await generateAvatarUrl({
|
||||
domain,
|
||||
id: subdomain,
|
||||
});
|
||||
if (!avatarUrl?.startsWith("http")) {
|
||||
avatarUrl = null;
|
||||
}
|
||||
|
||||
const team = await Team.create(
|
||||
|
||||
@@ -192,7 +192,7 @@ describe("userProvisioner", () => {
|
||||
|
||||
it("should prefer isAdmin argument over defaultUserRole", async () => {
|
||||
const team = await buildTeam({
|
||||
defaultUserRole: "viewer",
|
||||
defaultUserRole: UserRole.Viewer,
|
||||
});
|
||||
const authenticationProviders = await team.$get("authenticationProviders");
|
||||
const authenticationProvider = authenticationProviders[0];
|
||||
|
||||
@@ -3,14 +3,13 @@ import compact from "lodash/compact";
|
||||
import flatten from "lodash/flatten";
|
||||
import isEqual from "lodash/isEqual";
|
||||
import uniq from "lodash/uniq";
|
||||
import { Node, DOMSerializer, Fragment, Mark } from "prosemirror-model";
|
||||
import { Node, DOMSerializer, Fragment } from "prosemirror-model";
|
||||
import * as React from "react";
|
||||
import { renderToString } from "react-dom/server";
|
||||
import styled, { ServerStyleSheet, ThemeProvider } from "styled-components";
|
||||
import { prosemirrorToYDoc } from "y-prosemirror";
|
||||
import * as Y from "yjs";
|
||||
import EditorContainer from "@shared/editor/components/Styles";
|
||||
import embeds from "@shared/editor/embeds";
|
||||
import GlobalStyles from "@shared/styles/globals";
|
||||
import light from "@shared/styles/theme";
|
||||
import { MentionType, ProsemirrorData } from "@shared/types";
|
||||
@@ -61,47 +60,7 @@ export class ProsemirrorHelper {
|
||||
);
|
||||
}
|
||||
|
||||
let node = parser.parse(input);
|
||||
|
||||
// in the editor embeds are created at runtime by converting links into
|
||||
// embeds where they match.Because we're converting to a CRDT structure on
|
||||
// the server we need to mimic this behavior.
|
||||
function urlsToEmbeds(node: Node): Node {
|
||||
if (node.type.name === "paragraph") {
|
||||
for (const textNode of node.content.content) {
|
||||
for (const embed of embeds) {
|
||||
if (
|
||||
textNode.text &&
|
||||
textNode.marks.some(
|
||||
(m: Mark) =>
|
||||
m.type.name === "link" && m.attrs.href === textNode.text
|
||||
) &&
|
||||
embed.matcher(textNode.text)
|
||||
) {
|
||||
return schema.nodes.embed.createAndFill({
|
||||
href: textNode.text,
|
||||
}) as Node;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (node.content) {
|
||||
const contentAsArray =
|
||||
node.content instanceof Fragment
|
||||
? node.content.content
|
||||
: node.content;
|
||||
// @ts-expect-error content
|
||||
node.content = Fragment.fromArray(contentAsArray.map(urlsToEmbeds));
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
if (node) {
|
||||
node = urlsToEmbeds(node);
|
||||
}
|
||||
|
||||
const node = parser.parse(input);
|
||||
return node ? prosemirrorToYDoc(node, fieldName) : new Y.Doc();
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import ExportJSONTask from "../tasks/ExportJSONTask";
|
||||
import ExportMarkdownZipTask from "../tasks/ExportMarkdownZipTask";
|
||||
import ImportJSONTask from "../tasks/ImportJSONTask";
|
||||
import ImportMarkdownZipTask from "../tasks/ImportMarkdownZipTask";
|
||||
import ImportNotionTask from "../tasks/ImportNotionTask";
|
||||
import BaseProcessor from "./BaseProcessor";
|
||||
|
||||
export default class FileOperationCreatedProcessor extends BaseProcessor {
|
||||
@@ -25,11 +24,6 @@ export default class FileOperationCreatedProcessor extends BaseProcessor {
|
||||
fileOperationId: event.modelId,
|
||||
});
|
||||
break;
|
||||
case FileOperationFormat.Notion:
|
||||
await ImportNotionTask.schedule({
|
||||
fileOperationId: event.modelId,
|
||||
});
|
||||
break;
|
||||
case FileOperationFormat.JSON:
|
||||
await ImportJSONTask.schedule({
|
||||
fileOperationId: event.modelId,
|
||||
|
||||
@@ -3,7 +3,12 @@ import chunk from "lodash/chunk";
|
||||
import keyBy from "lodash/keyBy";
|
||||
import truncate from "lodash/truncate";
|
||||
import { Fragment, Node } from "prosemirror-model";
|
||||
import { CreateOptions, CreationAttributes, Transaction } from "sequelize";
|
||||
import {
|
||||
CreateOptions,
|
||||
CreationAttributes,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from "sequelize";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomElement } from "@shared/random";
|
||||
import { ImportInput, ImportTaskInput } from "@shared/schema";
|
||||
@@ -154,45 +159,59 @@ export default abstract class ImportsProcessor<
|
||||
* @returns Promise that resolves when mapping and persistence is completed.
|
||||
*/
|
||||
private async onProcessed(importModel: Import<T>, transaction: Transaction) {
|
||||
const { collections } = await this.createCollectionsAndDocuments({
|
||||
importModel,
|
||||
transaction,
|
||||
});
|
||||
|
||||
// Once all collections and documents are created, update collection's document structure.
|
||||
// This ensures the root documents have the whole subtree available in the structure.
|
||||
for (const collection of collections) {
|
||||
await Document.unscoped().findAllInBatches<Document>(
|
||||
{
|
||||
where: { parentDocumentId: null, collectionId: collection.id },
|
||||
order: [
|
||||
["createdAt", "DESC"],
|
||||
["id", "ASC"],
|
||||
],
|
||||
transaction,
|
||||
},
|
||||
async (documents) => {
|
||||
for (const document of documents) {
|
||||
await collection.addDocumentToStructure(document, 0, {
|
||||
save: false,
|
||||
silent: true,
|
||||
transaction,
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
await collection.save({ silent: true, transaction });
|
||||
}
|
||||
|
||||
importModel.state = ImportState.Completed;
|
||||
importModel.error = null; // unset any error from previous attempts.
|
||||
await importModel.saveWithCtx(
|
||||
createContext({
|
||||
user: importModel.createdBy,
|
||||
try {
|
||||
const { collections } = await this.createCollectionsAndDocuments({
|
||||
importModel,
|
||||
transaction,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
// Once all collections and documents are created, update collection's document structure.
|
||||
// This ensures the root documents have the whole subtree available in the structure.
|
||||
for (const collection of collections) {
|
||||
await Document.unscoped().findAllInBatches<Document>(
|
||||
{
|
||||
where: { parentDocumentId: null, collectionId: collection.id },
|
||||
order: [
|
||||
["createdAt", "DESC"],
|
||||
["id", "ASC"],
|
||||
],
|
||||
transaction,
|
||||
},
|
||||
async (documents) => {
|
||||
for (const document of documents) {
|
||||
await collection.addDocumentToStructure(document, 0, {
|
||||
save: false,
|
||||
silent: true,
|
||||
transaction,
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
await collection.save({ silent: true, transaction });
|
||||
}
|
||||
|
||||
importModel.state = ImportState.Completed;
|
||||
importModel.error = null; // unset any error from previous attempts.
|
||||
await importModel.saveWithCtx(
|
||||
createContext({
|
||||
user: importModel.createdBy,
|
||||
transaction,
|
||||
})
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof UniqueConstraintError) {
|
||||
Logger.error(
|
||||
"ImportsProcessor persistence failed due to unique constraint error",
|
||||
err,
|
||||
{
|
||||
fields: err.fields,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -304,6 +323,15 @@ export default abstract class ImportsProcessor<
|
||||
|
||||
const output = outputMap[externalId];
|
||||
|
||||
// Skip this item if it has no output (likely due to an error during processing)
|
||||
if (!output) {
|
||||
Logger.debug(
|
||||
"processor",
|
||||
`Skipping item with no output: ${externalId}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const collectionItem = importInput[externalId];
|
||||
|
||||
const attachments = await Attachment.findAll({
|
||||
@@ -444,7 +472,7 @@ export default abstract class ImportsProcessor<
|
||||
importInput: Record<string, ImportInput<any>[number]>;
|
||||
actorId: string;
|
||||
}): ProsemirrorDoc {
|
||||
// special case when the doc content is empty
|
||||
// special case when the doc content is empty.
|
||||
if (!content.content.length) {
|
||||
return content;
|
||||
}
|
||||
|
||||
@@ -170,7 +170,7 @@ export default abstract class APIImportTask<
|
||||
await importTask.save({ transaction });
|
||||
|
||||
const associatedImport = importTask.import;
|
||||
associatedImport.documentCount += importTask.input.length;
|
||||
associatedImport.documentCount += taskOutputWithReplacements.length;
|
||||
await associatedImport.saveWithCtx(
|
||||
createContext({
|
||||
user: associatedImport.createdBy,
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
/* eslint-disable @typescript-eslint/no-empty-function */
|
||||
import path from "path";
|
||||
import { FileOperation } from "@server/models";
|
||||
import { buildFileOperation } from "@server/test/factories";
|
||||
import ImportNotionTask from "./ImportNotionTask";
|
||||
|
||||
describe("ImportNotionTask", () => {
|
||||
it("should import successfully from a Markdown export", async () => {
|
||||
const fileOperation = await buildFileOperation();
|
||||
Object.defineProperty(fileOperation, "handle", {
|
||||
get() {
|
||||
return {
|
||||
path: path.resolve(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"test",
|
||||
"fixtures",
|
||||
"notion-markdown.zip"
|
||||
),
|
||||
cleanup: async () => {},
|
||||
};
|
||||
},
|
||||
});
|
||||
jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
|
||||
|
||||
const props = {
|
||||
fileOperationId: fileOperation.id,
|
||||
};
|
||||
|
||||
const task = new ImportNotionTask();
|
||||
const response = await task.perform(props);
|
||||
|
||||
expect(response.collections.size).toEqual(2);
|
||||
expect(response.documents.size).toEqual(6);
|
||||
expect(response.attachments.size).toEqual(1);
|
||||
|
||||
// Check that the image url was replaced in the text with a redirect
|
||||
const attachments = Array.from(response.attachments.values());
|
||||
const documents = Array.from(response.documents.values());
|
||||
expect(documents.map((d) => d.text).join("")).toContain(
|
||||
attachments[0].redirectUrl
|
||||
);
|
||||
});
|
||||
|
||||
it("should import successfully from a HTML export", async () => {
|
||||
const fileOperation = await buildFileOperation();
|
||||
Object.defineProperty(fileOperation, "handle", {
|
||||
get() {
|
||||
return {
|
||||
path: path.resolve(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"test",
|
||||
"fixtures",
|
||||
"notion-html.zip"
|
||||
),
|
||||
cleanup: async () => {},
|
||||
};
|
||||
},
|
||||
});
|
||||
jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
|
||||
|
||||
const props = {
|
||||
fileOperationId: fileOperation.id,
|
||||
};
|
||||
|
||||
const task = new ImportNotionTask();
|
||||
const response = await task.perform(props);
|
||||
|
||||
expect(response.collections.size).toEqual(2);
|
||||
expect(response.documents.size).toEqual(6);
|
||||
expect(response.attachments.size).toEqual(4);
|
||||
|
||||
// Check that the image url was replaced in the text with a redirect
|
||||
const attachments = Array.from(response.attachments.values());
|
||||
const attachment = attachments.find((att) =>
|
||||
att.key.endsWith("Screen_Shot_2022-04-21_at_2.23.26_PM.png")
|
||||
);
|
||||
|
||||
const documents = Array.from(response.documents.values());
|
||||
expect(documents.map((d) => d.text).join("")).toContain(
|
||||
attachment?.redirectUrl
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,354 +0,0 @@
|
||||
import path from "path";
|
||||
import fs from "fs-extra";
|
||||
import compact from "lodash/compact";
|
||||
import escapeRegExp from "lodash/escapeRegExp";
|
||||
import mime from "mime-types";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import documentImporter from "@server/commands/documentImporter";
|
||||
import { createContext } from "@server/context";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import { FileOperation, User } from "@server/models";
|
||||
import { sequelize } from "@server/storage/database";
|
||||
import ImportHelper, { FileTreeNode } from "@server/utils/ImportHelper";
|
||||
import ImportTask, { StructuredImportData } from "./ImportTask";
|
||||
|
||||
export default class ImportNotionTask extends ImportTask {
|
||||
public async parseData(
|
||||
dirPath: string,
|
||||
fileOperation: FileOperation
|
||||
): Promise<StructuredImportData> {
|
||||
const tree = await ImportHelper.toFileTree(dirPath);
|
||||
if (!tree) {
|
||||
throw new Error("Could not find valid content in zip file");
|
||||
}
|
||||
|
||||
// New Notion exports have a single folder with the name of the export, we must skip this
|
||||
// folder and go directly to the children.
|
||||
let parsed;
|
||||
if (
|
||||
tree.children.length === 1 &&
|
||||
tree.children[0].children.find((child) => child.title === "index")
|
||||
) {
|
||||
parsed = await this.parseFileTree(
|
||||
fileOperation,
|
||||
tree.children[0].children.filter((child) => child.title !== "index")
|
||||
);
|
||||
} else {
|
||||
parsed = await this.parseFileTree(fileOperation, tree.children);
|
||||
}
|
||||
|
||||
if (parsed.documents.length === 0 && parsed.collections.length === 1) {
|
||||
const collection = parsed.collections[0];
|
||||
const collectionId = uuidv4();
|
||||
if (collection.description) {
|
||||
parsed.documents.push({
|
||||
title: collection.name,
|
||||
icon: collection.icon,
|
||||
color: collection.color,
|
||||
path: "",
|
||||
text: String(collection.description),
|
||||
id: collection.id,
|
||||
externalId: collection.externalId,
|
||||
mimeType: "text/html",
|
||||
collectionId,
|
||||
});
|
||||
}
|
||||
|
||||
collection.name = "Notion";
|
||||
collection.icon = undefined;
|
||||
collection.color = undefined;
|
||||
collection.externalId = undefined;
|
||||
collection.description = undefined;
|
||||
collection.id = collectionId;
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the file structure from zipAsFileTree into documents,
|
||||
* collections, and attachments.
|
||||
*
|
||||
* @param fileOperation The file operation
|
||||
* @param tree An array of FileTreeNode representing root files in the zip
|
||||
* @returns A StructuredImportData object
|
||||
*/
|
||||
private async parseFileTree(
|
||||
fileOperation: FileOperation,
|
||||
tree: FileTreeNode[]
|
||||
): Promise<StructuredImportData> {
|
||||
const user = await User.findByPk(fileOperation.userId, {
|
||||
rejectOnEmpty: true,
|
||||
});
|
||||
|
||||
const output: StructuredImportData = {
|
||||
collections: [],
|
||||
documents: [],
|
||||
attachments: [],
|
||||
};
|
||||
|
||||
const parseNodeChildren = async (
|
||||
children: FileTreeNode[],
|
||||
collectionId: string,
|
||||
parentDocumentId?: string
|
||||
): Promise<void> => {
|
||||
await Promise.all(
|
||||
children.map(async (child) => {
|
||||
// Ignore the CSV's for databases upfront
|
||||
if (child.path.endsWith(".csv")) {
|
||||
return;
|
||||
}
|
||||
|
||||
const id = uuidv4();
|
||||
const match = child.title.match(this.NotionUUIDRegex);
|
||||
const name = child.title.replace(this.NotionUUIDRegex, "");
|
||||
const externalId = match ? match[0].trim() : undefined;
|
||||
|
||||
// If it's not a text file we're going to treat it as an attachment.
|
||||
const mimeType = mime.lookup(child.name);
|
||||
const isDocument =
|
||||
mimeType === "text/markdown" ||
|
||||
mimeType === "text/plain" ||
|
||||
mimeType === "text/html";
|
||||
|
||||
// If it's not a document and not a folder, treat it as an attachment
|
||||
if (!isDocument && mimeType) {
|
||||
output.attachments.push({
|
||||
id,
|
||||
name: child.name,
|
||||
path: child.path,
|
||||
mimeType,
|
||||
buffer: () => fs.readFile(child.path),
|
||||
externalId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
Logger.debug("task", `Processing ${name} as ${mimeType}`);
|
||||
|
||||
const { title, icon, text } = await sequelize.transaction(
|
||||
async (transaction) =>
|
||||
documentImporter({
|
||||
mimeType: mimeType || "text/markdown",
|
||||
fileName: name,
|
||||
content:
|
||||
child.children.length > 0
|
||||
? ""
|
||||
: await fs.readFile(child.path, "utf8"),
|
||||
user,
|
||||
ctx: createContext({ user, transaction }),
|
||||
})
|
||||
);
|
||||
|
||||
const existingDocumentIndex = output.documents.findIndex(
|
||||
(doc) => doc.externalId === externalId
|
||||
);
|
||||
|
||||
const existingDocument = output.documents[existingDocumentIndex];
|
||||
|
||||
// If there is an existing document with the same externalId that means
|
||||
// we've already parsed either a folder or a file referencing the same
|
||||
// document, as such we should merge.
|
||||
if (existingDocument) {
|
||||
if (existingDocument.text === "") {
|
||||
output.documents[existingDocumentIndex].text = text;
|
||||
}
|
||||
|
||||
await parseNodeChildren(
|
||||
child.children,
|
||||
collectionId,
|
||||
existingDocument.id
|
||||
);
|
||||
} else {
|
||||
output.documents.push({
|
||||
id,
|
||||
title,
|
||||
icon,
|
||||
text,
|
||||
collectionId,
|
||||
parentDocumentId,
|
||||
path: child.path,
|
||||
mimeType: mimeType || "text/markdown",
|
||||
externalId,
|
||||
});
|
||||
await parseNodeChildren(child.children, collectionId, id);
|
||||
}
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const replaceInternalLinksAndImages = (text: string) => {
|
||||
// Find if there are any images in this document
|
||||
const imagesInText = this.parseImages(text);
|
||||
|
||||
for (const image of imagesInText) {
|
||||
const name = path.basename(image.src);
|
||||
const attachment = output.attachments.find(
|
||||
(att) =>
|
||||
att.path.endsWith(image.src) ||
|
||||
encodeURI(att.path).endsWith(image.src)
|
||||
);
|
||||
|
||||
if (!attachment) {
|
||||
if (!image.src.startsWith("http")) {
|
||||
Logger.info(
|
||||
"task",
|
||||
`Could not find referenced attachment with name ${name} and src ${image.src}`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
text = text.replace(
|
||||
new RegExp(escapeRegExp(image.src), "g"),
|
||||
`<<${attachment.id}>>`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// With Notion's HTML import, images sometimes come wrapped in anchor tags
|
||||
// This isn't supported in Outline's editor, so we need to strip them.
|
||||
text = text.replace(/\[!\[([^[]+)]/g, "![]");
|
||||
|
||||
// Find if there are any links in this document pointing to other documents
|
||||
const internalLinksInText = this.parseInternalLinks(text);
|
||||
|
||||
// For each link update to the standardized format of <<documentId>>
|
||||
// instead of a relative or absolute URL within the original zip file.
|
||||
for (const link of internalLinksInText) {
|
||||
const doc = output.documents.find(
|
||||
(doc) => doc.externalId === link.externalId
|
||||
);
|
||||
|
||||
if (!doc) {
|
||||
Logger.info(
|
||||
"task",
|
||||
`Could not find referenced document with externalId ${link.externalId}`
|
||||
);
|
||||
} else {
|
||||
text = text.replace(link.href, `<<${doc.id}>>`);
|
||||
}
|
||||
}
|
||||
|
||||
return text;
|
||||
};
|
||||
|
||||
// All nodes in the root level should become collections
|
||||
for (const node of tree) {
|
||||
const match = node.title.match(this.NotionUUIDRegex);
|
||||
const name = node.title.replace(this.NotionUUIDRegex, "");
|
||||
const externalId = match ? match[0].trim() : undefined;
|
||||
const mimeType = mime.lookup(node.name);
|
||||
|
||||
const existingCollectionIndex = output.collections.findIndex(
|
||||
(collection) => collection.externalId === externalId
|
||||
);
|
||||
const existingCollection = output.collections[existingCollectionIndex];
|
||||
const collectionId = existingCollection?.id || uuidv4();
|
||||
let description;
|
||||
|
||||
// Root level docs become the descriptions of collections
|
||||
if (
|
||||
mimeType === "text/markdown" ||
|
||||
mimeType === "text/plain" ||
|
||||
mimeType === "text/html"
|
||||
) {
|
||||
const { text } = await sequelize.transaction(async (transaction) =>
|
||||
documentImporter({
|
||||
mimeType,
|
||||
fileName: name,
|
||||
content: await fs.readFile(node.path, "utf8"),
|
||||
user,
|
||||
ctx: createContext({ user, transaction }),
|
||||
})
|
||||
);
|
||||
|
||||
description = text;
|
||||
} else if (node.children.length > 0) {
|
||||
await parseNodeChildren(node.children, collectionId);
|
||||
} else {
|
||||
Logger.debug("task", `Unhandled file in zip: ${node.path}`, {
|
||||
fileOperationId: fileOperation.id,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existingCollectionIndex !== -1) {
|
||||
if (description) {
|
||||
output.collections[existingCollectionIndex].description = description;
|
||||
}
|
||||
} else {
|
||||
output.collections.push({
|
||||
id: collectionId,
|
||||
name,
|
||||
description,
|
||||
externalId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const document of output.documents) {
|
||||
document.text = replaceInternalLinksAndImages(document.text);
|
||||
}
|
||||
|
||||
for (const collection of output.collections) {
|
||||
if (typeof collection.description === "string") {
|
||||
collection.description = replaceInternalLinksAndImages(
|
||||
collection.description
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts internal links from a markdown document, taking into account the
|
||||
* externalId of the document, which is part of the link title.
|
||||
*
|
||||
* @param text The markdown text to parse
|
||||
* @returns An array of internal links
|
||||
*/
|
||||
private parseInternalLinks(
|
||||
text: string
|
||||
): { title: string; href: string; externalId: string }[] {
|
||||
return compact(
|
||||
[...text.matchAll(this.NotionLinkRegex)].map((match) => ({
|
||||
title: match[1],
|
||||
href: match[2],
|
||||
externalId: match[3],
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts images from the markdown document
|
||||
*
|
||||
* @param text The markdown text to parse
|
||||
* @returns An array of internal links
|
||||
*/
|
||||
private parseImages(text: string): { alt: string; src: string }[] {
|
||||
return compact(
|
||||
[...text.matchAll(this.ImageRegex)].map((match) => ({
|
||||
alt: match[1],
|
||||
src: match[2],
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Regex to find markdown images of all types
|
||||
*/
|
||||
private ImageRegex =
|
||||
/!\[(?<alt>[^\][]*?)]\((?<filename>[^\][]*?)(?=“|\))“?(?<title>[^\][”]+)?”?\)/g;
|
||||
|
||||
/**
|
||||
* Regex to find markdown links containing ID's that look like UUID's with the
|
||||
* "-"'s removed, Notion's externalId format.
|
||||
*/
|
||||
private NotionLinkRegex = /\[([^[]+)]\((.*?([0-9a-fA-F]{32})\..*?)\)/g;
|
||||
|
||||
/**
|
||||
* Regex to find Notion document UUID's in the title of a document.
|
||||
*/
|
||||
private NotionUUIDRegex =
|
||||
/\s([0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}|[0-9a-fA-F]{32})$/;
|
||||
}
|
||||
@@ -597,6 +597,7 @@ router.post(
|
||||
createdById: user.id,
|
||||
},
|
||||
transaction,
|
||||
hooks: false,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import queryString from "query-string";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomElement } from "@shared/random";
|
||||
import { NotificationEventType } from "@shared/types";
|
||||
import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper";
|
||||
import {
|
||||
buildCollection,
|
||||
buildDocument,
|
||||
@@ -697,3 +698,40 @@ describe("#notifications.update_all", () => {
|
||||
expect(body.data.total).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#notifications.unsubscribe", () => {
|
||||
it("should allow unsubscribe with valid token", async () => {
|
||||
const user = await buildUser();
|
||||
const token = NotificationSettingsHelper.unsubscribeToken(
|
||||
user.id,
|
||||
NotificationEventType.UpdateDocument
|
||||
);
|
||||
|
||||
const res = await server.get(
|
||||
`/api/notifications.unsubscribe?userId=${user.id}&token=${token}&eventType=documents.update&follow=true`,
|
||||
{
|
||||
redirect: "manual",
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(302);
|
||||
expect(res.headers.get("location")).toContain(
|
||||
"/settings/notifications?success"
|
||||
);
|
||||
|
||||
const events = (await user.reload()).notificationSettings;
|
||||
expect(events).not.toContain("documents.update");
|
||||
});
|
||||
|
||||
it("should not allow unsubscribe with invalid token", async () => {
|
||||
const user = await buildUser();
|
||||
|
||||
const res = await server.get(
|
||||
`/api/notifications.unsubscribe?userId=${user.id}&token=invalid-token&eventType=documents.update&follow=true`,
|
||||
{
|
||||
redirect: "manual",
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(302);
|
||||
expect(res.headers.get("location")).toContain("?notice=invalid-auth");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,7 +53,7 @@ const handleUnsubscribe = async (
|
||||
});
|
||||
|
||||
user.setNotificationEventType(eventType, false);
|
||||
await user.save();
|
||||
await user.save({ transaction });
|
||||
ctx.redirect(`${user.team.url}/settings/notifications?success`);
|
||||
};
|
||||
|
||||
|
||||
Vendored
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,9 +0,0 @@
|
||||
import { generateAvatarUrl } from "./avatars";
|
||||
|
||||
it("should return clearbit url if available", async () => {
|
||||
const url = await generateAvatarUrl({
|
||||
id: "google",
|
||||
domain: "google.com",
|
||||
});
|
||||
expect(url).toBe("https://logo.clearbit.com/google.com");
|
||||
});
|
||||
@@ -1,28 +0,0 @@
|
||||
import crypto from "crypto";
|
||||
import fetch from "./fetch";
|
||||
|
||||
export async function generateAvatarUrl({
|
||||
id,
|
||||
domain,
|
||||
}: {
|
||||
id: string;
|
||||
domain?: string;
|
||||
}) {
|
||||
// attempt to get logo from Clearbit API. If one doesn't exist then
|
||||
// fall back to using tiley to generate a placeholder logo
|
||||
const hash = crypto.createHash("sha256");
|
||||
hash.update(id);
|
||||
let cbResponse, cbUrl;
|
||||
|
||||
if (domain) {
|
||||
cbUrl = `https://logo.clearbit.com/${domain}`;
|
||||
|
||||
try {
|
||||
cbResponse = await fetch(cbUrl);
|
||||
} catch (err) {
|
||||
// okay
|
||||
}
|
||||
}
|
||||
|
||||
return cbUrl && cbResponse && cbResponse.status === 200 ? cbUrl : null;
|
||||
}
|
||||
@@ -28,9 +28,10 @@ export function getCellAttrs(dom: HTMLElement | string): Attrs {
|
||||
const widthAttr = dom.getAttribute("data-colwidth");
|
||||
const widths =
|
||||
widthAttr && /^\d+(,\d+)*$/.test(widthAttr)
|
||||
? widthAttr.split(",").map((s) => Number(s))
|
||||
? widthAttr.split(",").map(Number)
|
||||
: null;
|
||||
const colspan = Number(dom.getAttribute("colspan") || 1);
|
||||
|
||||
return {
|
||||
colspan,
|
||||
rowspan: Number(dom.getAttribute("rowspan") || 1),
|
||||
@@ -63,10 +64,11 @@ export function setCellAttrs(node: Node): Attrs {
|
||||
}
|
||||
if (node.attrs.colwidth) {
|
||||
if (isBrowser) {
|
||||
attrs["data-colwidth"] = node.attrs.colwidth.join(",");
|
||||
attrs["data-colwidth"] = node.attrs.colwidth.map(parseInt).join(",");
|
||||
} else {
|
||||
attrs.style =
|
||||
(attrs.style ?? "") + `min-width: ${node.attrs.colwidth}px;`;
|
||||
(attrs.style ?? "") +
|
||||
`min-width: ${parseInt(node.attrs.colwidth[0])}px;`;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ export default class Code extends Mark {
|
||||
excludes: "mention placeholder highlight",
|
||||
parseDOM: [{ tag: "code", preserveWhitespace: true }],
|
||||
toDOM: () => ["code", { class: "inline", spellCheck: "false" }],
|
||||
code: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -98,8 +98,6 @@ export default class TableCell extends Node {
|
||||
}
|
||||
}
|
||||
|
||||
// For all other cases, preserve the slice as is to maintain table structure
|
||||
// including column widths and other attributes
|
||||
return slice;
|
||||
},
|
||||
handleDOMEvents: {
|
||||
|
||||
@@ -903,9 +903,6 @@
|
||||
"{{ count }} document imported_plural": "{{ count }} documents imported",
|
||||
"You can import a zip file that was previously exported from an Outline installation – collections, documents, and images will be imported. In Outline, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.": "You can import a zip file that was previously exported from an Outline installation – collections, documents, and images will be imported. In Outline, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.",
|
||||
"Drag and drop the zip file from the Markdown export option in {{appName}}, or click to upload": "Drag and drop the zip file from the Markdown export option in {{appName}}, or click to upload",
|
||||
"Where do I find the file?": "Where do I find the file?",
|
||||
"In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability.": "In Notion, click <em>Settings & Members</em> in the left sidebar and open Settings. Look for the Export section, and click <em>Export all workspace content</em>. Choose <em>HTML</em> as the format for the best data compatability.",
|
||||
"Drag and drop the zip file from Notion's HTML export option, or click to upload": "Drag and drop the zip file from Notion's HTML export option, or click to upload",
|
||||
"Last active": "Last active",
|
||||
"Guest": "Guest",
|
||||
"Shared by": "Shared by",
|
||||
|
||||
Reference in New Issue
Block a user