Merge pull request FlowiseAI#774 from vectara/sentence-config

Add sentence config option to Vectara

Author: HenryHengZJ

packages/components/nodes/vectorstores/Vectara_Existing/Vectara_Existing.ts

@@ -1,6 +1,6 @@
 import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { VectaraStore, VectaraLibArgs, VectaraFilter } from 'langchain/vectorstores/vectara'
+import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara'
 
 class VectaraExisting_VectorStores implements INode {
     label: string
@@ -40,9 +40,27 @@ class VectaraExisting_VectorStores implements INode {
                 additionalParams: true,
                 optional: true
             },
+            {
+                label: 'Sentences Before',
+                name: 'sentencesBefore',
+                description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Sentences After',
+                name: 'sentencesAfter',
+                description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
             {
                 label: 'Lambda',
                 name: 'lambda',
+                description:
+                    'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
                 type: 'number',
                 additionalParams: true,
                 optional: true
@@ -77,6 +95,8 @@ class VectaraExisting_VectorStores implements INode {
         const corpusId = getCredentialParam('corpusID', credentialData, nodeData)
 
         const vectaraMetadataFilter = nodeData.inputs?.filter as string
+        const sentencesBefore = nodeData.inputs?.sentencesBefore as number
+        const sentencesAfter = nodeData.inputs?.sentencesAfter as number
         const lambda = nodeData.inputs?.lambda as number
         const output = nodeData.outputs?.output as string
         const topK = nodeData.inputs?.topK as string
@@ -92,6 +112,11 @@ class VectaraExisting_VectorStores implements INode {
         if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
         if (lambda) vectaraFilter.lambda = lambda
 
+        const vectaraContextConfig: VectaraContextConfig = {}
+        if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
+        if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
+        vectaraFilter.contextConfig = vectaraContextConfig
+
         const vectorStore = new VectaraStore(vectaraArgs)
 
         if (output === 'retriever') {

packages/components/nodes/vectorstores/Vectara_Upsert/Vectara_Upsert.ts

@@ -1,7 +1,7 @@
 import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
 import { Embeddings } from 'langchain/embeddings/base'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { VectaraStore, VectaraLibArgs, VectaraFilter } from 'langchain/vectorstores/vectara'
+import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig } from 'langchain/vectorstores/vectara'
 import { Document } from 'langchain/document'
 import { flatten } from 'lodash'
 
@@ -49,9 +49,27 @@ class VectaraUpsert_VectorStores implements INode {
                 additionalParams: true,
                 optional: true
             },
+            {
+                label: 'Sentences Before',
+                name: 'sentencesBefore',
+                description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Sentences After',
+                name: 'sentencesAfter',
+                description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            },
             {
                 label: 'Lambda',
                 name: 'lambda',
+                description:
+                    'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
                 type: 'number',
                 additionalParams: true,
                 optional: true
@@ -88,6 +106,8 @@ class VectaraUpsert_VectorStores implements INode {
         const docs = nodeData.inputs?.document as Document[]
         const embeddings = {} as Embeddings
         const vectaraMetadataFilter = nodeData.inputs?.filter as string
+        const sentencesBefore = nodeData.inputs?.sentencesBefore as number
+        const sentencesAfter = nodeData.inputs?.sentencesAfter as number
         const lambda = nodeData.inputs?.lambda as number
         const output = nodeData.outputs?.output as string
         const topK = nodeData.inputs?.topK as string
@@ -103,6 +123,11 @@ class VectaraUpsert_VectorStores implements INode {
         if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
         if (lambda) vectaraFilter.lambda = lambda
 
+        const vectaraContextConfig: VectaraContextConfig = {}
+        if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
+        if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
+        vectaraFilter.contextConfig = vectaraContextConfig
+
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {

packages/components/package.json

@@ -40,7 +40,7 @@
         "google-auth-library": "^9.0.0",
         "graphql": "^16.6.0",
         "html-to-text": "^9.0.5",
-        "langchain": "^0.0.122",
+        "langchain": "^0.0.126",
         "linkifyjs": "^4.1.1",
         "mammoth": "^1.5.1",
         "moment": "^2.29.3",