Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/GithubLoader AdditionalParams #854

Merged
merged 3 commits into from
Aug 30, 2023
Merged

Feature/GithubLoader AdditionalParams #854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fda4963
add additional params
HenryHengZJ Aug 30, 2023
b2627ca
update steps
HenryHengZJ Aug 30, 2023
6372cb9
update marketplace
HenryHengZJ Aug 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion packages/components/nodes/documentloaders/Github/Github.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class Github_DocumentLoaders implements INode {
constructor() {
this.label = 'Github'
this.name = 'github'
this.version = 1.0
this.version = 2.0
this.type = 'Document'
this.icon = 'github.png'
this.category = 'Document Loaders'
Expand Down Expand Up @@ -51,6 +51,34 @@ class Github_DocumentLoaders implements INode {
type: 'boolean',
optional: true
},
{
label: 'Max Concurrency',
name: 'maxConcurrency',
type: 'number',
step: 1,
optional: true,
additionalParams: true
},
{
label: 'Ignore Paths',
name: 'ignorePath',
description: 'An array of paths to be ignored',
placeholder: `["*.md"]`,
type: 'string',
rows: 4,
optional: true,
additionalParams: true
},
{
label: 'Max Retries',
name: 'maxRetries',
description:
'The maximum number of retries that can be made for a single call, with an exponential backoff between each attempt. Defaults to 2.',
type: 'number',
step: 1,
optional: true,
additionalParams: true
},
{
label: 'Text Splitter',
name: 'textSplitter',
Expand All @@ -73,6 +101,9 @@ class Github_DocumentLoaders implements INode {
const recursive = nodeData.inputs?.recursive as boolean
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata
const maxConcurrency = nodeData.inputs?.maxConcurrency as string
const maxRetries = nodeData.inputs?.maxRetries as string
const ignorePath = nodeData.inputs?.ignorePath as string

const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const accessToken = getCredentialParam('accessToken', credentialData, nodeData)
Expand All @@ -84,6 +115,9 @@ class Github_DocumentLoaders implements INode {
}

if (accessToken) githubOptions.accessToken = accessToken
if (maxConcurrency) githubOptions.maxConcurrency = parseInt(maxConcurrency, 10)
if (maxRetries) githubOptions.maxRetries = parseInt(maxRetries, 10)
if (ignorePath) githubOptions.ignorePaths = JSON.parse(ignorePath)

const loader = new GithubRepoLoader(repoLink, githubOptions)
const docs = textSplitter ? await loader.loadAndSplit(textSplitter) : await loader.load()
Expand Down
31 changes: 30 additions & 1 deletion packages/server/marketplaces/chatflows/Flowise Docs QnA.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@
"id": "github_0",
"label": "Github",
"name": "github",
"version": 1,
"version": 2,
"type": "Document",
"baseClasses": ["Document"],
"category": "Document Loaders",
Expand Down Expand Up @@ -301,6 +301,35 @@
"optional": true,
"id": "github_0-input-recursive-boolean"
},
{
"label": "Max Concurrency",
"name": "maxConcurrency",
"type": "number",
"step": 1,
"optional": true,
"additionalParams": true,
"id": "github_0-input-maxConcurrency-number"
},
{
"label": "Ignore Paths",
"name": "ignorePath",
"type": "string",
"description": "An array of paths to be ignored",
"placeholder": "[\"*.md\"]",
"rows": 4,
"optional": true,
"additionalParams": true,
"id": "github_0-input-ignorePath-string"
},
{
"label": "Max Retries",
"name": "maxRetries",
"description": "The maximum number of retries that can be made for a single call, with an exponential backoff between each attempt. Defaults to 2.",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "github_0-input-maxRetries-number"
},
{
"label": "Metadata",
"name": "metadata",
Expand Down