Skip to content

Commit 1431143

Browse files
HenryHengZJHenryHengZJ
committed
Merge branch 'main' into feature/ZepVS
# Conflicts: # packages/components/package.json
2 parents 02cb080 + a6d3e11 commit 1431143

File tree

11 files changed

+470
-228
lines changed

11 files changed

+470
-228
lines changed

‎packages/components/nodes/agents/ConversationalRetrievalAgent/ConversationalRetrievalAgent.ts

+4-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { flatten } from 'lodash'
55
import { BaseChatMemory } from 'langchain/memory'
66
import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
77

8+
const defaultMessage = `Do your best to answer the questions. Feel free to use any tools available to look up relevant information, only if necessary.`
9+
810
class ConversationalRetrievalAgent_Agents implements INode {
911
label: string
1012
name: string
@@ -46,6 +48,7 @@ class ConversationalRetrievalAgent_Agents implements INode {
4648
label: 'System Message',
4749
name: 'systemMessage',
4850
type: 'string',
51+
default: defaultMessage,
4952
rows: 4,
5053
optional: true,
5154
additionalParams: true
@@ -65,7 +68,7 @@ class ConversationalRetrievalAgent_Agents implements INode {
6568
agentType: 'openai-functions',
6669
verbose: process.env.DEBUG === 'true' ? true : false,
6770
agentArgs: {
68-
prefix: systemMessage ?? `You are a helpful AI assistant.`
71+
prefix: systemMessage ?? defaultMessage
6972
},
7073
returnIntermediateSteps: true
7174
})

‎packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts

+37-15
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ import { DataSource } from 'typeorm'
55
import { SqlDatabase } from 'langchain/sql_db'
66
import { BaseLanguageModel } from 'langchain/base_language'
77
import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
8+
import { DataSourceOptions } from 'typeorm/data-source'
9+
10+
type DatabaseType = 'sqlite' | 'postgres' | 'mssql' | 'mysql'
811

912
class SqlDatabaseChain_Chains implements INode {
1013
label: string
@@ -38,36 +41,48 @@ class SqlDatabaseChain_Chains implements INode {
3841
type: 'options',
3942
options: [
4043
{
41-
label: 'SQlite',
44+
label: 'SQLite',
4245
name: 'sqlite'
46+
},
47+
{
48+
label: 'PostgreSQL',
49+
name: 'postgres'
50+
},
51+
{
52+
label: 'MSSQL',
53+
name: 'mssql'
54+
},
55+
{
56+
label: 'MySQL',
57+
name: 'mysql'
4358
}
4459
],
4560
default: 'sqlite'
4661
},
4762
{
48-
label: 'Database File Path',
49-
name: 'dbFilePath',
63+
label: 'Connection string or file path (sqlite only)',
64+
name: 'url',
5065
type: 'string',
51-
placeholder: 'C:/Users/chinook.db'
66+
placeholder: '1270.0.0.1:5432/chinook'
5267
}
5368
]
5469
}
5570

5671
async init(nodeData: INodeData): Promise<any> {
57-
const databaseType = nodeData.inputs?.database as 'sqlite'
72+
const databaseType = nodeData.inputs?.database as DatabaseType
5873
const model = nodeData.inputs?.model as BaseLanguageModel
59-
const dbFilePath = nodeData.inputs?.dbFilePath
74+
const url = nodeData.inputs?.url
6075

61-
const chain = await getSQLDBChain(databaseType, dbFilePath, model)
76+
const chain = await getSQLDBChain(databaseType, url, model)
6277
return chain
6378
}
6479

6580
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
66-
const databaseType = nodeData.inputs?.database as 'sqlite'
81+
const databaseType = nodeData.inputs?.database as DatabaseType
6782
const model = nodeData.inputs?.model as BaseLanguageModel
68-
const dbFilePath = nodeData.inputs?.dbFilePath
83+
const url = nodeData.inputs?.url
6984

70-
const chain = await getSQLDBChain(databaseType, dbFilePath, model)
85+
const chain = await getSQLDBChain(databaseType, url, model)
7186
const loggerHandler = new ConsoleCallbackHandler(options.logger)
7287

7388
if (options.socketIO && options.socketIOClientId) {
@@ -81,11 +96,18 @@ class SqlDatabaseChain_Chains implements INode {
8196
}
8297
}
8398

84-
const getSQLDBChain = async (databaseType: 'sqlite', dbFilePath: string, llm: BaseLanguageModel) => {
85-
const datasource = new DataSource({
86-
type: databaseType,
87-
database: dbFilePath
88-
})
99+
const getSQLDBChain = async (databaseType: DatabaseType, url: string, llm: BaseLanguageModel) => {
100+
const datasource = new DataSource(
101+
databaseType === 'sqlite'
102+
? {
103+
type: databaseType,
104+
database: url
105+
}
106+
: ({
107+
type: databaseType,
108+
url: url
109+
} as DataSourceOptions)
110+
)
89111

90112
const db = await SqlDatabase.fromDataSourceParams({
91113
appDataSource: datasource

‎packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts

+19-1
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,13 @@ class ChatOpenAI_ChatModels implements INode {
125125
type: 'string',
126126
optional: true,
127127
additionalParams: true
128+
},
129+
{
130+
label: 'BaseOptions',
131+
name: 'baseOptions',
132+
type: 'json',
133+
optional: true,
134+
additionalParams: true
128135
}
129136
]
130137
}
@@ -139,6 +146,7 @@ class ChatOpenAI_ChatModels implements INode {
139146
const timeout = nodeData.inputs?.timeout as string
140147
const streaming = nodeData.inputs?.streaming as boolean
141148
const basePath = nodeData.inputs?.basepath as string
149+
const baseOptions = nodeData.inputs?.baseOptions
142150

143151
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
144152
const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
@@ -156,8 +164,18 @@ class ChatOpenAI_ChatModels implements INode {
156164
if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty)
157165
if (timeout) obj.timeout = parseInt(timeout, 10)
158166

167+
let parsedBaseOptions: any | undefined = undefined
168+
169+
if (baseOptions) {
170+
try {
171+
parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
172+
} catch (exception) {
173+
throw new Error("Invalid JSON in the ChatOpenAI's BaseOptions: " + exception)
174+
}
175+
}
159176
const model = new ChatOpenAI(obj, {
160-
basePath
177+
basePath,
178+
baseOptions: parsedBaseOptions
161179
})
162180
return model
163181
}

‎packages/components/nodes/documentloaders/Cheerio/Cheerio.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ class Cheerio_DocumentLoaders implements INode {
6464
additionalParams: true,
6565
description:
6666
'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.',
67-
warning: `Retreiving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)`
67+
warning: `Retrieving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)`
6868
},
6969
{
7070
label: 'Metadata',

‎packages/components/nodes/documentloaders/Folder/Folder.ts

+34-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,40 @@ class Folder_DocumentLoaders implements INode {
6161
'.csv': (path) => new CSVLoader(path),
6262
'.docx': (path) => new DocxLoader(path),
6363
// @ts-ignore
64-
'.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') })
64+
'.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }),
65+
'.aspx': (path) => new TextLoader(path),
66+
'.asp': (path) => new TextLoader(path),
67+
'.cpp': (path) => new TextLoader(path), // C++
68+
'.c': (path) => new TextLoader(path),
69+
'.cs': (path) => new TextLoader(path),
70+
'.css': (path) => new TextLoader(path),
71+
'.go': (path) => new TextLoader(path), // Go
72+
'.h': (path) => new TextLoader(path), // C++ Header files
73+
'.java': (path) => new TextLoader(path), // Java
74+
'.js': (path) => new TextLoader(path), // JavaScript
75+
'.less': (path) => new TextLoader(path), // Less files
76+
'.ts': (path) => new TextLoader(path), // TypeScript
77+
'.php': (path) => new TextLoader(path), // PHP
78+
'.proto': (path) => new TextLoader(path), // Protocol Buffers
79+
'.python': (path) => new TextLoader(path), // Python
80+
'.py': (path) => new TextLoader(path), // Python
81+
'.rst': (path) => new TextLoader(path), // reStructuredText
82+
'.ruby': (path) => new TextLoader(path), // Ruby
83+
'.rb': (path) => new TextLoader(path), // Ruby
84+
'.rs': (path) => new TextLoader(path), // Rust
85+
'.scala': (path) => new TextLoader(path), // Scala
86+
'.sc': (path) => new TextLoader(path), // Scala
87+
'.scss': (path) => new TextLoader(path), // Sass
88+
'.sol': (path) => new TextLoader(path), // Solidity
89+
'.sql': (path) => new TextLoader(path), //SQL
90+
'.swift': (path) => new TextLoader(path), // Swift
91+
'.markdown': (path) => new TextLoader(path), // Markdown
92+
'.md': (path) => new TextLoader(path), // Markdown
93+
'.tex': (path) => new TextLoader(path), // LaTeX
94+
'.ltx': (path) => new TextLoader(path), // LaTeX
95+
'.html': (path) => new TextLoader(path), // HTML
96+
'.vb': (path) => new TextLoader(path), // Visual Basic
97+
'.xml': (path) => new TextLoader(path) // XML
6598
})
6699
let docs = []
67100

‎packages/components/nodes/documentloaders/Playwright/Playwright.ts

+62-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { INode, INodeData, INodeParams } from '../../../src/Interface'
22
import { TextSplitter } from 'langchain/text_splitter'
3-
import { PlaywrightWebBaseLoader } from 'langchain/document_loaders/web/playwright'
3+
import { Browser, Page, PlaywrightWebBaseLoader, PlaywrightWebBaseLoaderOptions } from 'langchain/document_loaders/web/playwright'
44
import { test } from 'linkifyjs'
55
import { webCrawl, xmlScrape } from '../../../src'
66

@@ -64,7 +64,45 @@ class Playwright_DocumentLoaders implements INode {
6464
additionalParams: true,
6565
description:
6666
'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.',
67-
warning: `Retreiving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)`
67+
warning: `Retrieving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)`
68+
},
69+
{
70+
label: 'Wait Until',
71+
name: 'waitUntilGoToOption',
72+
type: 'options',
73+
description: 'Select a go to wait until option',
74+
options: [
75+
{
76+
label: 'Load',
77+
name: 'load',
78+
description: 'Consider operation to be finished when the load event is fired.'
79+
},
80+
{
81+
label: 'DOM Content Loaded',
82+
name: 'domcontentloaded',
83+
description: 'Consider operation to be finished when the DOMContentLoaded event is fired.'
84+
},
85+
{
86+
label: 'Network Idle',
87+
name: 'networkidle',
88+
description: 'Navigation is finished when there are no more connections for at least 500 ms.'
89+
},
90+
{
91+
label: 'Commit',
92+
name: 'commit',
93+
description: 'Consider operation to be finished when network response is received and the document started loading.'
94+
}
95+
],
96+
optional: true,
97+
additionalParams: true
98+
},
99+
{
100+
label: 'Wait for selector to load',
101+
name: 'waitForSelector',
102+
type: 'string',
103+
optional: true,
104+
additionalParams: true,
105+
description: 'CSS selectors like .div or #div'
68106
},
69107
{
70108
label: 'Metadata',
@@ -81,6 +119,8 @@ class Playwright_DocumentLoaders implements INode {
81119
const metadata = nodeData.inputs?.metadata
82120
const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string
83121
let limit = nodeData.inputs?.limit as string
122+
let waitUntilGoToOption = nodeData.inputs?.waitUntilGoToOption as 'load' | 'domcontentloaded' | 'networkidle' | 'commit' | undefined
123+
let waitForSelector = nodeData.inputs?.waitForSelector as string
84124

85125
let url = nodeData.inputs?.url as string
86126
url = url.trim()
@@ -91,7 +131,26 @@ class Playwright_DocumentLoaders implements INode {
91131
async function playwrightLoader(url: string): Promise<any> {
92132
try {
93133
let docs = []
94-
const loader = new PlaywrightWebBaseLoader(url)
134+
const config: PlaywrightWebBaseLoaderOptions = {
135+
launchOptions: {
136+
args: ['--no-sandbox'],
137+
headless: true
138+
}
139+
}
140+
if (waitUntilGoToOption) {
141+
config['gotoOptions'] = {
142+
waitUntil: waitUntilGoToOption
143+
}
144+
}
145+
if (waitForSelector) {
146+
config['evaluate'] = async (page: Page, _: Browser): Promise<string> => {
147+
await page.waitForSelector(waitForSelector)
148+
149+
const result = await page.evaluate(() => document.body.innerHTML)
150+
return result
151+
}
152+
}
153+
const loader = new PlaywrightWebBaseLoader(url, config)
95154
if (textSplitter) {
96155
docs = await loader.loadAndSplit(textSplitter)
97156
} else {

‎packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts

+59-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import { INode, INodeData, INodeParams } from '../../../src/Interface'
22
import { TextSplitter } from 'langchain/text_splitter'
3-
import { PuppeteerWebBaseLoader } from 'langchain/document_loaders/web/puppeteer'
3+
import { Browser, Page, PuppeteerWebBaseLoader, PuppeteerWebBaseLoaderOptions } from 'langchain/document_loaders/web/puppeteer'
44
import { test } from 'linkifyjs'
55
import { webCrawl, xmlScrape } from '../../../src'
6+
import { PuppeteerLifeCycleEvent } from 'puppeteer'
67

78
class Puppeteer_DocumentLoaders implements INode {
89
label: string
@@ -64,7 +65,45 @@ class Puppeteer_DocumentLoaders implements INode {
6465
additionalParams: true,
6566
description:
6667
'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.',
67-
warning: `Retreiving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)`
68+
warning: `Retrieving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)`
69+
},
70+
{
71+
label: 'Wait Until',
72+
name: 'waitUntilGoToOption',
73+
type: 'options',
74+
description: 'Select a go to wait until option',
75+
options: [
76+
{
77+
label: 'Load',
78+
name: 'load',
79+
description: `When the initial HTML document's DOM has been loaded and parsed`
80+
},
81+
{
82+
label: 'DOM Content Loaded',
83+
name: 'domcontentloaded',
84+
description: `When the complete HTML document's DOM has been loaded and parsed`
85+
},
86+
{
87+
label: 'Network Idle 0',
88+
name: 'networkidle0',
89+
description: 'Navigation is finished when there are no more than 0 network connections for at least 500 ms'
90+
},
91+
{
92+
label: 'Network Idle 2',
93+
name: 'networkidle2',
94+
description: 'Navigation is finished when there are no more than 2 network connections for at least 500 ms'
95+
}
96+
],
97+
optional: true,
98+
additionalParams: true
99+
},
100+
{
101+
label: 'Wait for selector to load',
102+
name: 'waitForSelector',
103+
type: 'string',
104+
optional: true,
105+
additionalParams: true,
106+
description: 'CSS selectors like .div or #div'
68107
},
69108
{
70109
label: 'Metadata',
@@ -81,6 +120,8 @@ class Puppeteer_DocumentLoaders implements INode {
81120
const metadata = nodeData.inputs?.metadata
82121
const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string
83122
let limit = nodeData.inputs?.limit as string
123+
let waitUntilGoToOption = nodeData.inputs?.waitUntilGoToOption as PuppeteerLifeCycleEvent
124+
let waitForSelector = nodeData.inputs?.waitForSelector as string
84125

85126
let url = nodeData.inputs?.url as string
86127
url = url.trim()
@@ -91,12 +132,26 @@ class Puppeteer_DocumentLoaders implements INode {
91132
async function puppeteerLoader(url: string): Promise<any> {
92133
try {
93134
let docs = []
94-
const loader = new PuppeteerWebBaseLoader(url, {
135+
const config: PuppeteerWebBaseLoaderOptions = {
95136
launchOptions: {
96137
args: ['--no-sandbox'],
97138
headless: 'new'
98139
}
99-
})
140+
}
141+
if (waitUntilGoToOption) {
142+
config['gotoOptions'] = {
143+
waitUntil: waitUntilGoToOption
144+
}
145+
}
146+
if (waitForSelector) {
147+
config['evaluate'] = async (page: Page, _: Browser): Promise<string> => {
148+
await page.waitForSelector(waitForSelector)
149+
150+
const result = await page.evaluate(() => document.body.innerHTML)
151+
return result
152+
}
153+
}
154+
const loader = new PuppeteerWebBaseLoader(url, config)
100155
if (textSplitter) {
101156
docs = await loader.loadAndSplit(textSplitter)
102157
} else {

0 commit comments

Comments
 (0)