Skip to content

Commit b6f9269

Browse files
authored
Merge pull request #591 from FlowiseAI/feature/Notion
Feature/Update Notion Loader
2 parents 6de237a + b5b9b9c commit b6f9269

File tree

6 files changed

+114
-22
lines changed

6 files changed

+114
-22
lines changed

packages/components/nodes/documentloaders/NotionDB/NotionDB.ts packages/components/nodes/documentloaders/Notion/NotionDB.ts

+14-20
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { INode, INodeData, INodeParams } from '../../../src/Interface'
22
import { TextSplitter } from 'langchain/text_splitter'
3-
import { NotionDBLoader, NotionDBLoaderParams } from 'langchain/document_loaders/web/notiondb'
3+
import { NotionAPILoader, NotionAPILoaderOptions } from 'langchain/document_loaders/web/notionapi'
44

55
class NotionDB_DocumentLoaders implements INode {
66
label: string
@@ -18,7 +18,7 @@ class NotionDB_DocumentLoaders implements INode {
1818
this.type = 'Document'
1919
this.icon = 'notion.png'
2020
this.category = 'Document Loaders'
21-
this.description = 'Load data from Notion Database ID'
21+
this.description = 'Load data from Notion Database (each row is a separate document with all properties as metadata)'
2222
this.baseClasses = [this.type]
2323
this.inputs = [
2424
{
@@ -27,13 +27,6 @@ class NotionDB_DocumentLoaders implements INode {
2727
type: 'TextSplitter',
2828
optional: true
2929
},
30-
{
31-
label: 'Notion Database Id',
32-
name: 'databaseId',
33-
type: 'string',
34-
description:
35-
'If your URL looks like - https://www.notion.so/<long_hash_1>?v=<long_hash_2>, then <long_hash_1> is the database ID'
36-
},
3730
{
3831
label: 'Notion Integration Token',
3932
name: 'notionIntegrationToken',
@@ -42,10 +35,10 @@ class NotionDB_DocumentLoaders implements INode {
4235
'You can find integration token <a target="_blank" href="https://developers.notion.com/docs/create-a-notion-integration#step-1-create-an-integration">here</a>'
4336
},
4437
{
45-
label: 'Page Size Limit',
46-
name: 'pageSizeLimit',
47-
type: 'number',
48-
default: 10
38+
label: 'Notion Database Id',
39+
name: 'databaseId',
40+
type: 'string',
41+
description: 'If your URL looks like - https://www.notion.so/abcdefh?v=long_hash_2, then abcdefh is the database ID'
4942
},
5043
{
5144
label: 'Metadata',
@@ -60,16 +53,17 @@ class NotionDB_DocumentLoaders implements INode {
6053
async init(nodeData: INodeData): Promise<any> {
6154
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
6255
const databaseId = nodeData.inputs?.databaseId as string
63-
const notionIntegrationToken = nodeData.inputs?.notionIntegrationToken as string
64-
const pageSizeLimit = nodeData.inputs?.pageSizeLimit as string
6556
const metadata = nodeData.inputs?.metadata
57+
const notionIntegrationToken = nodeData.inputs?.notionIntegrationToken as string
6658

67-
const obj: NotionDBLoaderParams = {
68-
pageSizeLimit: pageSizeLimit ? parseInt(pageSizeLimit, 10) : 10,
69-
databaseId,
70-
notionIntegrationToken
59+
const obj: NotionAPILoaderOptions = {
60+
clientOptions: {
61+
auth: notionIntegrationToken
62+
},
63+
id: databaseId,
64+
type: 'database'
7165
}
72-
const loader = new NotionDBLoader(obj)
66+
const loader = new NotionAPILoader(obj)
7367

7468
let docs = []
7569
if (textSplitter) {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import { INode, INodeData, INodeParams } from '../../../src/Interface'
2+
import { TextSplitter } from 'langchain/text_splitter'
3+
import { NotionAPILoader, NotionAPILoaderOptions } from 'langchain/document_loaders/web/notionapi'
4+
5+
class NotionPage_DocumentLoaders implements INode {
6+
label: string
7+
name: string
8+
description: string
9+
type: string
10+
icon: string
11+
category: string
12+
baseClasses: string[]
13+
inputs: INodeParams[]
14+
15+
constructor() {
16+
this.label = 'Notion Page'
17+
this.name = 'notionPage'
18+
this.type = 'Document'
19+
this.icon = 'notion.png'
20+
this.category = 'Document Loaders'
21+
this.description = 'Load data from Notion Page (including child pages all as separate documents)'
22+
this.baseClasses = [this.type]
23+
this.inputs = [
24+
{
25+
label: 'Text Splitter',
26+
name: 'textSplitter',
27+
type: 'TextSplitter',
28+
optional: true
29+
},
30+
{
31+
label: 'Notion Integration Token',
32+
name: 'notionIntegrationToken',
33+
type: 'password',
34+
description:
35+
'You can find integration token <a target="_blank" href="https://developers.notion.com/docs/create-a-notion-integration#step-1-create-an-integration">here</a>'
36+
},
37+
{
38+
label: 'Notion Page Id',
39+
name: 'pageId',
40+
type: 'string',
41+
description:
42+
'The last The 32 char hex in the url path. For example: https://www.notion.so/skarard/LangChain-Notion-API-b34ca03f219c4420a6046fc4bdfdf7b4, b34ca03f219c4420a6046fc4bdfdf7b4 is the Page ID'
43+
},
44+
{
45+
label: 'Metadata',
46+
name: 'metadata',
47+
type: 'json',
48+
optional: true,
49+
additionalParams: true
50+
}
51+
]
52+
}
53+
54+
async init(nodeData: INodeData): Promise<any> {
55+
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
56+
const pageId = nodeData.inputs?.pageId as string
57+
const metadata = nodeData.inputs?.metadata
58+
const notionIntegrationToken = nodeData.inputs?.notionIntegrationToken as string
59+
60+
const obj: NotionAPILoaderOptions = {
61+
clientOptions: {
62+
auth: notionIntegrationToken
63+
},
64+
id: pageId,
65+
type: 'page'
66+
}
67+
const loader = new NotionAPILoader(obj)
68+
69+
let docs = []
70+
if (textSplitter) {
71+
docs = await loader.loadAndSplit(textSplitter)
72+
} else {
73+
docs = await loader.load()
74+
}
75+
76+
if (metadata) {
77+
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
78+
let finaldocs = []
79+
for (const doc of docs) {
80+
const newdoc = {
81+
...doc,
82+
metadata: {
83+
...doc.metadata,
84+
...parsedMetadata
85+
}
86+
}
87+
finaldocs.push(newdoc)
88+
}
89+
return finaldocs
90+
}
91+
92+
return docs
93+
}
94+
}
95+
96+
module.exports = { nodeClass: NotionPage_DocumentLoaders }
Binary file not shown.

packages/components/package.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"@dqbd/tiktoken": "^1.0.7",
2121
"@getzep/zep-js": "^0.4.1",
2222
"@huggingface/inference": "^2.6.1",
23+
"@notionhq/client": "^2.2.8",
2324
"@opensearch-project/opensearch": "^1.2.0",
2425
"@pinecone-database/pinecone": "^0.0.12",
2526
"@qdrant/js-client-rest": "^1.2.2",
@@ -41,8 +42,10 @@
4142
"linkifyjs": "^4.1.1",
4243
"mammoth": "^1.5.1",
4344
"moment": "^2.29.3",
45+
"mysql2": "^3.5.1",
4446
"node-fetch": "^2.6.11",
4547
"node-html-markdown": "^1.3.0",
48+
"notion-to-md": "^3.1.1",
4649
"pdf-parse": "^1.1.1",
4750
"pdfjs-dist": "^3.7.107",
4851
"playwright": "^1.35.0",
@@ -52,8 +55,7 @@
5255
"srt-parser-2": "^1.2.3",
5356
"vm2": "^3.9.19",
5457
"weaviate-ts-client": "^1.1.0",
55-
"ws": "^8.9.0",
56-
"mysql2": "^3.5.1"
58+
"ws": "^8.9.0"
5759
},
5860
"devDependencies": {
5961
"@types/gulp": "4.0.9",

0 commit comments

Comments
 (0)