Skip to content

Commit 491c4be

Browse files
HenryHengZJHenryHengZJ
authored
Merge pull request #540 from apeng-singlestore/feature/singlestore
Add singlestore upsert and existing
2 parents 3530d29 + 258496e commit 491c4be

File tree

6 files changed

+341
-2
lines changed

6 files changed

+341
-2
lines changed

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
"*.{js,jsx,ts,tsx,json,md}": "eslint --fix"
2929
},
3030
"devDependencies": {
31-
"turbo": "1.7.4",
3231
"@babel/preset-env": "^7.19.4",
3332
"@babel/preset-typescript": "7.18.6",
3433
"@types/express": "^4.17.13",
@@ -48,6 +47,7 @@
4847
"pretty-quick": "^3.1.3",
4948
"rimraf": "^3.0.2",
5049
"run-script-os": "^1.1.6",
50+
"turbo": "1.7.4",
5151
"typescript": "^4.8.4"
5252
},
5353
"engines": {

‎packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
2+
import { Embeddings } from 'langchain/embeddings/base'
3+
import { getBaseClasses } from '../../../src/utils'
4+
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore'
5+
6+
class SingleStoreExisting_VectorStores implements INode {
7+
label: string
8+
name: string
9+
description: string
10+
type: string
11+
icon: string
12+
category: string
13+
baseClasses: string[]
14+
inputs: INodeParams[]
15+
outputs: INodeOutputsValue[]
16+
17+
constructor() {
18+
this.label = 'SingleStore Load Existing Table'
19+
this.name = 'singlestoreExisting'
20+
this.type = 'SingleStore'
21+
this.icon = 'singlestore.svg'
22+
this.category = 'Vector Stores'
23+
this.description = 'Load existing document from SingleStore'
24+
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
25+
this.inputs = [
26+
{
27+
label: 'Embeddings',
28+
name: 'embeddings',
29+
type: 'Embeddings'
30+
},
31+
{
32+
label: 'Host',
33+
name: 'host',
34+
type: 'string'
35+
},
36+
{
37+
label: 'User',
38+
name: 'user',
39+
type: 'string'
40+
},
41+
{
42+
label: 'Password',
43+
name: 'password',
44+
type: 'password'
45+
},
46+
{
47+
label: 'Database',
48+
name: 'database',
49+
type: 'string'
50+
},
51+
{
52+
label: 'Table Name',
53+
name: 'tableName',
54+
type: 'string',
55+
placeholder: 'embeddings',
56+
additionalParams: true,
57+
optional: true
58+
},
59+
{
60+
label: 'Content Column Name',
61+
name: 'contentColumnName',
62+
type: 'string',
63+
placeholder: 'content',
64+
additionalParams: true,
65+
optional: true
66+
},
67+
{
68+
label: 'Vector Column Name',
69+
name: 'vectorColumnName',
70+
type: 'string',
71+
placeholder: 'vector',
72+
additionalParams: true,
73+
optional: true
74+
},
75+
{
76+
label: 'Metadata Column Name',
77+
name: 'metadataColumnName',
78+
type: 'string',
79+
placeholder: 'metadata',
80+
additionalParams: true,
81+
optional: true
82+
},
83+
{
84+
label: 'Top K',
85+
name: 'topK',
86+
placeholder: '4',
87+
type: 'number',
88+
additionalParams: true,
89+
optional: true
90+
}
91+
]
92+
this.outputs = [
93+
{
94+
label: 'SingleStore Retriever',
95+
name: 'retriever',
96+
baseClasses: this.baseClasses
97+
},
98+
{
99+
label: 'SingleStore Vector Store',
100+
name: 'vectorStore',
101+
baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
102+
}
103+
]
104+
}
105+
106+
async init(nodeData: INodeData): Promise<any> {
107+
const singleStoreConnectionConfig = {
108+
connectionOptions: {
109+
host: nodeData.inputs?.host as string,
110+
port: 3306,
111+
user: nodeData.inputs?.user as string,
112+
password: nodeData.inputs?.password as string,
113+
database: nodeData.inputs?.database as string
114+
},
115+
...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
116+
...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
117+
...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
118+
...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
119+
} as SingleStoreVectorStoreConfig
120+
121+
const embeddings = nodeData.inputs?.embeddings as Embeddings
122+
const output = nodeData.outputs?.output as string
123+
const topK = nodeData.inputs?.topK as string
124+
const k = topK ? parseInt(topK, 10) : 4
125+
126+
let vectorStore: SingleStoreVectorStore
127+
128+
vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
129+
130+
if (output === 'retriever') {
131+
const retriever = vectorStore.asRetriever(k)
132+
return retriever
133+
} else if (output === 'vectorStore') {
134+
;(vectorStore as any).k = k
135+
return vectorStore
136+
}
137+
return vectorStore
138+
}
139+
}
140+
141+
module.exports = { nodeClass: SingleStoreExisting_VectorStores }

‎packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg

+20
Loading

‎packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts

+157
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
2+
import { Embeddings } from 'langchain/embeddings/base'
3+
import { Document } from 'langchain/document'
4+
import { getBaseClasses } from '../../../src/utils'
5+
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore'
6+
import { flatten } from 'lodash'
7+
8+
class SingleStoreUpsert_VectorStores implements INode {
9+
label: string
10+
name: string
11+
description: string
12+
type: string
13+
icon: string
14+
category: string
15+
baseClasses: string[]
16+
inputs: INodeParams[]
17+
outputs: INodeOutputsValue[]
18+
19+
constructor() {
20+
this.label = 'SingleStore Upsert Document'
21+
this.name = 'singlestoreUpsert'
22+
this.type = 'SingleStore'
23+
this.icon = 'singlestore.svg'
24+
this.category = 'Vector Stores'
25+
this.description = 'Upsert documents to SingleStore'
26+
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
27+
this.inputs = [
28+
{
29+
label: 'Document',
30+
name: 'document',
31+
type: 'Document',
32+
list: true
33+
},
34+
{
35+
label: 'Embeddings',
36+
name: 'embeddings',
37+
type: 'Embeddings'
38+
},
39+
{
40+
label: 'Host',
41+
name: 'host',
42+
type: 'string'
43+
},
44+
{
45+
label: 'User',
46+
name: 'user',
47+
type: 'string'
48+
},
49+
{
50+
label: 'Password',
51+
name: 'password',
52+
type: 'password'
53+
},
54+
{
55+
label: 'Database',
56+
name: 'database',
57+
type: 'string'
58+
},
59+
{
60+
label: 'Table Name',
61+
name: 'tableName',
62+
type: 'string',
63+
placeholder: 'embeddings',
64+
additionalParams: true,
65+
optional: true
66+
},
67+
{
68+
label: 'Content Column Name',
69+
name: 'contentColumnName',
70+
type: 'string',
71+
placeholder: 'content',
72+
additionalParams: true,
73+
optional: true
74+
},
75+
{
76+
label: 'Vector Column Name',
77+
name: 'vectorColumnName',
78+
type: 'string',
79+
placeholder: 'vector',
80+
additionalParams: true,
81+
optional: true
82+
},
83+
{
84+
label: 'Metadata Column Name',
85+
name: 'metadataColumnName',
86+
type: 'string',
87+
placeholder: 'metadata',
88+
additionalParams: true,
89+
optional: true
90+
},
91+
{
92+
label: 'Top K',
93+
name: 'topK',
94+
placeholder: '4',
95+
type: 'number',
96+
additionalParams: true,
97+
optional: true
98+
}
99+
]
100+
this.outputs = [
101+
{
102+
label: 'SingleStore Retriever',
103+
name: 'retriever',
104+
baseClasses: this.baseClasses
105+
},
106+
{
107+
label: 'SingleStore Vector Store',
108+
name: 'vectorStore',
109+
baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
110+
}
111+
]
112+
}
113+
114+
async init(nodeData: INodeData): Promise<any> {
115+
const singleStoreConnectionConfig = {
116+
connectionOptions: {
117+
host: nodeData.inputs?.host as string,
118+
port: 3306,
119+
user: nodeData.inputs?.user as string,
120+
password: nodeData.inputs?.password as string,
121+
database: nodeData.inputs?.database as string
122+
},
123+
...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
124+
...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
125+
...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
126+
...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
127+
} as SingleStoreVectorStoreConfig
128+
129+
const docs = nodeData.inputs?.document as Document[]
130+
const embeddings = nodeData.inputs?.embeddings as Embeddings
131+
const output = nodeData.outputs?.output as string
132+
const topK = nodeData.inputs?.topK as string
133+
const k = topK ? parseInt(topK, 10) : 4
134+
135+
const flattenDocs = docs && docs.length ? flatten(docs) : []
136+
const finalDocs = []
137+
for (let i = 0; i < flattenDocs.length; i += 1) {
138+
finalDocs.push(new Document(flattenDocs[i]))
139+
}
140+
141+
let vectorStore: SingleStoreVectorStore
142+
143+
vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
144+
vectorStore.addDocuments.bind(vectorStore)(finalDocs)
145+
146+
if (output === 'retriever') {
147+
const retriever = vectorStore.asRetriever(k)
148+
return retriever
149+
} else if (output === 'vectorStore') {
150+
;(vectorStore as any).k = k
151+
return vectorStore
152+
}
153+
return vectorStore
154+
}
155+
}
156+
157+
module.exports = { nodeClass: SingleStoreUpsert_VectorStores }

‎packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg

+20
Loading

‎packages/components/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@
5252
"srt-parser-2": "^1.2.3",
5353
"vm2": "^3.9.19",
5454
"weaviate-ts-client": "^1.1.0",
55-
"ws": "^8.9.0"
55+
"ws": "^8.9.0",
56+
"mysql2": "^3.5.1"
5657
},
5758
"devDependencies": {
5859
"@types/gulp": "4.0.9",

0 commit comments

Comments
 (0)