Skip to content

Commit 75500d7

Browse files
committed
fix: sublist items in columns (sillsdev#70)
Also improves numbered lists by numbering them sequentially
1 parent 05f65ea commit 75500d7

11 files changed

+207
-177
lines changed

package-lock.json

Lines changed: 6 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
"markdown-table": "^2.0.0",
3636
"node-fetch": "2.6.6",
3737
"notion-client": "^4",
38-
"notion-to-md": "2.5.5",
38+
"notion-to-md": "3.1.1",
3939
"path": "^0.12.7",
4040
"ts-node": "^10.2.1",
4141
"sanitize-filename": "^1.6.3"

src/config/default.docunotion.config.ts

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import { standardColumnListTransformer } from "../plugins/ColumnListTransformer"
66
import { standardColumnTransformer } from "../plugins/ColumnTransformer";
77
import { standardEscapeHtmlBlockModifier } from "../plugins/EscapeHtmlBlockModifier";
88
import { standardHeadingTransformer } from "../plugins/HeadingTransformer";
9-
import { standardNumberedListTransformer } from "../plugins/NumberedListTransformer";
109
import { standardTableTransformer } from "../plugins/TableTransformer";
1110
import { standardVideoTransformer } from "../plugins/VideoTransformer";
1211
import { standardExternalLinkConversion } from "../plugins/externalLinks";
@@ -25,7 +24,6 @@ const defaultConfig: IDocuNotionConfig = {
2524
standardImageTransformer,
2625
standardCalloutTransformer,
2726
standardTableTransformer,
28-
standardNumberedListTransformer,
2927
standardVideoTransformer,
3028

3129
// Link modifiers, which are special because they can read metadata from all the pages in order to figure out the correct url

src/plugins/ColumnTransformer.spec.ts

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
import { NotionBlock } from "../types";
2+
import { blocksToMarkdown } from "./pluginTestRun";
3+
import { standardColumnTransformer } from "./ColumnTransformer";
4+
5+
// Even though we can set up most tests with our own children
6+
// so that we aren't relying on real data from Notion,
7+
// we can't prevent the notion-to-md library from making an API call
8+
// every time it processes a block with has_children:true.
9+
// So for these tests with children, we need any valid API key.
10+
const runTestsWhichRequireAnyValidApiKey =
11+
!!process.env.DOCU_NOTION_INTEGRATION_TOKEN;
12+
13+
// To test grandchildren, we can't get around notion-to-md making an API call
14+
// to get real children. So we need a specific notion record.
15+
// For that reason, we don't try to run these tests unless the user changes this flag.
16+
// But it is an important test; grandchildren in columns were broken.
17+
// See https://github.com/sillsdev/docu-notion/issues/70.
18+
const runManualTestsWhichRequireSpecificNotionRecords = false;
19+
20+
const columnBlock = {
21+
object: "block",
22+
id: "e6d2d7b7-b1ed-464a-86d2-bb5f6be78a03",
23+
has_children: true,
24+
type: "column",
25+
column: {},
26+
} as unknown as NotionBlock;
27+
async function getResults(children: NotionBlock[]) {
28+
return await blocksToMarkdown(
29+
{ plugins: [standardColumnTransformer] },
30+
[columnBlock],
31+
undefined,
32+
children,
33+
process.env.DOCU_NOTION_INTEGRATION_TOKEN
34+
);
35+
}
36+
37+
const columnWrapperStart =
38+
"<div class='notion-column' style=\\{\\{width: '.*?'\\}\\}>\\n\\n";
39+
const columnWrapperEnd =
40+
"\\n\\n<\\/div><div className='notion-spacer'><\\/div>";
41+
42+
if (runTestsWhichRequireAnyValidApiKey) {
43+
columnBlock.has_children = true;
44+
45+
test("requires API key - column with paragraph", async () => {
46+
const results = await getResults([getTestParagraphBlock()]);
47+
expect(results).toMatch(
48+
new RegExp(
49+
`${columnWrapperStart}\\s*?my paragraph\\s*?${columnWrapperEnd}`
50+
)
51+
);
52+
}, 20000);
53+
54+
test("requires API key - column with two paragraphs", async () => {
55+
const results = await getResults([
56+
getTestParagraphBlock(1),
57+
getTestParagraphBlock(2),
58+
]);
59+
expect(results).toMatch(
60+
new RegExp(
61+
`${columnWrapperStart}\\s*?my paragraph 1\\s+?my paragraph 2\\s*?${columnWrapperEnd}`
62+
)
63+
);
64+
}, 20000);
65+
66+
test("requires API key - column with numbered list", async () => {
67+
const results = await getResults([
68+
getNumberedListItemBlock(1),
69+
getNumberedListItemBlock(2),
70+
]);
71+
expect(results).toMatch(
72+
new RegExp(
73+
`${columnWrapperStart}\\s*?1\\. Numbered list item 1\\s+?2\\. Numbered list item 2\\s*?${columnWrapperEnd}`,
74+
"s"
75+
)
76+
);
77+
}, 20000);
78+
79+
if (runManualTestsWhichRequireSpecificNotionRecords) {
80+
test("manual test - requires specific notion record and API key - column with numbered list with sublist", async () => {
81+
const realNumberedListBlock = getNumberedListItemBlock(1);
82+
realNumberedListBlock.id = "ca08d14b-9b70-4f6f-9d17-9fd74b57afeb";
83+
realNumberedListBlock.has_children = true;
84+
85+
const results = await getResults([realNumberedListBlock]);
86+
expect(results).toMatch(
87+
new RegExp(
88+
`${columnWrapperStart}\\s*?1\\. Numbered list item 1\\s+?- unordered sub-bullet\\s*?${columnWrapperEnd}`,
89+
"s"
90+
)
91+
);
92+
}, 20000);
93+
}
94+
} else {
95+
// This test prevents an error when runTestsWhichRequireAnyValidApiKey is false
96+
// due to having a test suite with no tests.
97+
test("no column transformer tests were run because there is no API key provided", () => {
98+
expect(true).toBe(true);
99+
});
100+
}
101+
102+
function getNumberedListItemBlock(identifier?: number) {
103+
const content = identifier
104+
? `Numbered list item ${identifier}`
105+
: `Numbered list item`;
106+
return {
107+
object: "block",
108+
type: "numbered_list_item",
109+
numbered_list_item: {
110+
rich_text: [
111+
{
112+
type: "text",
113+
text: { content: content },
114+
annotations: {
115+
code: false,
116+
},
117+
plain_text: content,
118+
},
119+
],
120+
},
121+
} as unknown as NotionBlock;
122+
}
123+
124+
function getTestParagraphBlock(identifier?: number) {
125+
const content = identifier ? `my paragraph ${identifier}` : `my paragraph`;
126+
return {
127+
object: "block",
128+
type: "paragraph",
129+
paragraph: {
130+
rich_text: [
131+
{
132+
type: "text",
133+
text: {
134+
content: content,
135+
},
136+
annotations: {
137+
code: false,
138+
},
139+
plain_text: content,
140+
},
141+
],
142+
},
143+
} as unknown as NotionBlock;
144+
}

src/plugins/ColumnTransformer.ts

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
import { NotionAPI } from "notion-client";
22
import { NotionToMarkdown } from "notion-to-md";
3-
import { ListBlockChildrenResponseResult } from "notion-to-md/build/types";
3+
import {
4+
ListBlockChildrenResponseResult,
5+
MdBlock,
6+
} from "notion-to-md/build/types";
47
import { IGetBlockChildrenFn, IPlugin } from "./pluginTypes";
58
import { executeWithRateLimitAndRetries } from "../pull";
9+
import { NotionBlock } from "../types";
610

711
export const standardColumnTransformer: IPlugin = {
812
name: "standardColumnTransformer",
@@ -30,22 +34,21 @@ async function notionColumnToMarkdown(
3034

3135
if (!has_children) return "";
3236

33-
const children = await getBlockChildren(id);
34-
35-
const childrenPromises = children.map(
36-
async column => await notionToMarkdown.blockToMarkdown(column)
37+
const columnChildren: NotionBlock[] = await getBlockChildren(id);
38+
const childrenMdBlocksArray: MdBlock[][] = await Promise.all(
39+
columnChildren.map(
40+
async child => await notionToMarkdown.blocksToMarkdown([child])
41+
)
42+
);
43+
const childrenMarkdown = childrenMdBlocksArray.map(
44+
mdBlockArray => notionToMarkdown.toMarkdownString(mdBlockArray).parent
3745
);
38-
39-
const childrenStrings: string[] = await Promise.all(childrenPromises);
4046

4147
const columnWidth = await getColumnWidth(block);
42-
43-
// note: it would look better in the markup with \n, but that
44-
// causes notion-to-md to give us ":::A" instead of \n for some reason.
4548
return (
46-
`<div class='notion-column' style={{width: '${columnWidth}'}}>\n\n${childrenStrings.join(
47-
"\n\n"
48-
)}\n\n</div>` +
49+
`<div class='notion-column' style={{width: '${columnWidth}'}}>\n\n${childrenMarkdown.join(
50+
"\n"
51+
)}\n</div>` +
4952
// Spacer between columns. CSS takes care of hiding this for the last column
5053
// and when the screen is too narrow for multiple columns.
5154
`<div className='notion-spacer'></div>`

src/plugins/NumberedListTransformer.spec.ts

Lines changed: 0 additions & 78 deletions
This file was deleted.

src/plugins/NumberedListTransformer.ts

Lines changed: 0 additions & 67 deletions
This file was deleted.

0 commit comments

Comments
 (0)