FEATURE: Add Multi Modal Capabilities to Flowise

CONTRIBUTING-ZH.md

@@ -125,6 +125,7 @@ Flowise 支持不同的环境变量来配置您的实例。您可以在 `package
 | FLOWISE_PASSWORD            | 登录密码                                               | 字符串                                          |                                     |
 | FLOWISE_FILE_SIZE_LIMIT     | 上传文件大小限制                                       | 字符串                                          | 50mb                                |
 | DEBUG                       | 打印组件的日志                                         | 布尔值                                          |                                     |
+| BLOB_STORAGE_PATH           | 存储位置                                               | 字符串                                          | `your-home-dir/.flowise/storage`    |
 | LOG_PATH                    | 存储日志文件的位置                                     | 字符串                                          | `your-path/Flowise/logs`            |
 | LOG_LEVEL                   | 日志的不同级别                                         | 枚举字符串: `error`, `info`, `verbose`, `debug` | `info`                              |
 | APIKEY_PATH                 | 存储 API 密钥的位置                                    | 字符串                                          | `your-path/Flowise/packages/server` |

CONTRIBUTING.md

@@ -129,6 +129,7 @@ Flowise support different environment variables to configure your instance. You
 | FLOWISE_PASSWORD            | Password to login                                                            | String                                           |                                     |
 | FLOWISE_FILE_SIZE_LIMIT     | Upload File Size Limit                                                       | String                                           | 50mb                                |
 | DEBUG                       | Print logs from components                                                   | Boolean                                          |                                     |
+| BLOB_STORAGE_PATH           | Location where uploaded files are stored                                     | String                                           | `your-home-dir/.flowise/storage`    |
 | LOG_PATH                    | Location where log files are stored                                          | String                                           | `your-path/Flowise/logs`            |
 | LOG_LEVEL                   | Different levels of logs                                                     | Enum String: `error`, `info`, `verbose`, `debug` | `info`                              |
 | APIKEY_PATH                 | Location where api keys are saved                                            | String                                           | `your-path/Flowise/packages/server` |

docker/.env.example

@@ -3,6 +3,7 @@ DATABASE_PATH=/root/.flowise
 APIKEY_PATH=/root/.flowise
 SECRETKEY_PATH=/root/.flowise
 LOG_PATH=/root/.flowise/logs
+BLOB_STORAGE_PATH=/root/.flowise/storage
 
 # CORS_ORIGINS="*"
 # IFRAME_ORIGINS="*"

docker/README.md

@@ -31,5 +31,6 @@ If you like to persist your data (flows, logs, apikeys, credentials), set these
 -   APIKEY_PATH=/root/.flowise
 -   LOG_PATH=/root/.flowise/logs
 -   SECRETKEY_PATH=/root/.flowise
+-   BLOB_STORAGE_PATH=/root/.flowise/storage
 
 Flowise also support different environment variables to configure your instance. Read [more](https://docs.flowiseai.com/environment-variables)

docker/docker-compose.yml

@@ -26,6 +26,7 @@ services:
             - FLOWISE_SECRETKEY_OVERWRITE=${FLOWISE_SECRETKEY_OVERWRITE}
             - LOG_LEVEL=${LOG_LEVEL}
             - LOG_PATH=${LOG_PATH}
+            - BLOB_STORAGE_PATH=${BLOB_STORAGE_PATH}
             - DISABLE_FLOWISE_TELEMETRY=${DISABLE_FLOWISE_TELEMETRY}
         ports:
             - '${PORT}:${PORT}'

packages/components/credentials/AssemblyAI.credential.ts

@@ -0,0 +1,23 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class AssemblyAIApi implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'AssemblyAI API'
+        this.name = 'assemblyAIApi'
+        this.version = 1.0
+        this.inputs = [
+            {
+                label: 'AssemblyAI Api Key',
+                name: 'assemblyAIApiKey',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: AssemblyAIApi }

packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts

@@ -4,13 +4,15 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'
 import { ChainValues } from '@langchain/core/utils/types'
 import { AgentStep } from '@langchain/core/agents'
-import { renderTemplate } from '@langchain/core/prompts'
+import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
 import { RunnableSequence } from '@langchain/core/runnables'
 import { ChatConversationalAgent } from 'langchain/agents'
 import { getBaseClasses } from '../../../src/utils'
 import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
 import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { AgentExecutor } from '../../../src/agents'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
 
 const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI.
 
@@ -81,12 +83,18 @@ class ConversationalAgent_Agents implements INode {
     }
 
     async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
-        return prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
+        return prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
     }
 
     async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
         const memory = nodeData.inputs?.memory as FlowiseMemory
-        const executor = await prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory)
+
+        const executor = await prepareAgent(
+            nodeData,
+            options,
+            { sessionId: this.sessionId, chatId: options.chatId, input },
+            options.chatHistory
+        )
 
         const loggerHandler = new ConsoleCallbackHandler(options.logger)
         const callbacks = await additionalCallbacks(nodeData, options)
@@ -120,6 +128,7 @@ class ConversationalAgent_Agents implements INode {
 
 const prepareAgent = async (
     nodeData: INodeData,
+    options: ICommonObject,
     flowObj: { sessionId?: string; chatId?: string; input?: string },
     chatHistory: IMessage[] = []
 ) => {
@@ -131,11 +140,6 @@ const prepareAgent = async (
     const memoryKey = memory.memoryKey ? memory.memoryKey : 'chat_history'
     const inputKey = memory.inputKey ? memory.inputKey : 'input'
 
-    /** Bind a stop token to the model */
-    const modelWithStop = model.bind({
-        stop: ['\nObservation']
-    })
-
     const outputParser = ChatConversationalAgent.getDefaultOutputParser({
         llm: model,
         toolNames: tools.map((tool) => tool.name)
@@ -146,6 +150,41 @@ const prepareAgent = async (
         outputParser
     })
 
+    if (model instanceof ChatOpenAI) {
+        let humanImageMessages: HumanMessage[] = []
+        const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+        if (messageContent?.length) {
+            // Change model to gpt-4-vision
+            model.modelName = 'gpt-4-vision-preview'
+
+            // Change default max token to higher when using gpt-4-vision
+            model.maxTokens = 1024
+
+            for (const msg of messageContent) {
+                humanImageMessages.push(new HumanMessage({ content: [msg] }))
+            }
+
+            // Pop the `agent_scratchpad` MessagePlaceHolder
+            let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder
+
+            // Add the HumanMessage for images
+            prompt.promptMessages.push(...humanImageMessages)
+
+            // Add the `agent_scratchpad` MessagePlaceHolder back
+            prompt.promptMessages.push(messagePlaceholder)
+        } else {
+            // revert to previous values if image upload is empty
+            model.modelName = model.configuredModel
+            model.maxTokens = model.configuredMaxToken
+        }
+    }
+
+    /** Bind a stop token to the model */
+    const modelWithStop = model.bind({
+        stop: ['\nObservation']
+    })
+
     const runnableAgent = RunnableSequence.from([
         {
             [inputKey]: (i: { input: string; steps: AgentStep[] }) => i.input,
@@ -166,7 +205,7 @@ const prepareAgent = async (
         sessionId: flowObj?.sessionId,
         chatId: flowObj?.chatId,
         input: flowObj?.input,
-        verbose: process.env.DEBUG === 'true' ? true : false
+        verbose: process.env.DEBUG === 'true'
     })
 
     return executor

packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts

@@ -8,6 +8,10 @@ import { additionalCallbacks } from '../../../src/handler'
 import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
 import { createReactAgent } from '../../../src/agents'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { HumanMessage } from '@langchain/core/messages'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
+import { ChatPromptTemplate, HumanMessagePromptTemplate } from 'langchain/prompts'
 
 class MRKLAgentChat_Agents implements INode {
     label: string
@@ -61,18 +65,39 @@ class MRKLAgentChat_Agents implements INode {
         let tools = nodeData.inputs?.tools as Tool[]
         tools = flatten(tools)
 
-        const promptWithChat = await pull<PromptTemplate>('hwchase17/react-chat')
+        const prompt = await pull<PromptTemplate>('hwchase17/react-chat')
+        let chatPromptTemplate = undefined
+
+        if (model instanceof ChatOpenAI) {
+            const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+            if (messageContent?.length) {
+                // Change model to gpt-4-vision
+                model.modelName = 'gpt-4-vision-preview'
+
+                // Change default max token to higher when using gpt-4-vision
+                model.maxTokens = 1024
+
+                const oldTemplate = prompt.template as string
+                chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)])
+                chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent }))
+            } else {
+                // revert to previous values if image upload is empty
+                model.modelName = model.configuredModel
+                model.maxTokens = model.configuredMaxToken
+            }
+        }
 
         const agent = await createReactAgent({
             llm: model,
             tools,
-            prompt: promptWithChat
+            prompt: chatPromptTemplate ?? prompt
         })
 
         const executor = new AgentExecutor({
             agent,
             tools,
-            verbose: process.env.DEBUG === 'true' ? true : false
+            verbose: process.env.DEBUG === 'true'
         })
 
         const callbacks = await additionalCallbacks(nodeData, options)

packages/components/nodes/chains/ConversationChain/ConversationChain.ts

@@ -1,14 +1,16 @@
+import { ConversationChain } from 'langchain/chains'
 import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts'
-import { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import { RunnableSequence } from '@langchain/core/runnables'
 import { StringOutputParser } from '@langchain/core/output_parsers'
+import { HumanMessage } from '@langchain/core/messages'
 import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
-import { ConversationChain } from 'langchain/chains'
-import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
-import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
-import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
 import { formatResponse } from '../../outputparsers/OutputParserHelpers'
+import { addImagesToMessages } from '../../../src/multiModalUtils'
+import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI'
+import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
+import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 
 let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.`
 const inputKey = 'input'
@@ -86,12 +88,14 @@ class ConversationChain_Chains implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const chain = prepareChain(nodeData, this.sessionId, options.chatHistory)
+        const chain = prepareChain(nodeData, options, this.sessionId)
         return chain
     }
 
     async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
         const memory = nodeData.inputs?.memory
+
+        const chain = prepareChain(nodeData, options, this.sessionId)
         const moderations = nodeData.inputs?.inputModeration as Moderation[]
 
         if (moderations && moderations.length > 0) {
@@ -105,8 +109,6 @@ class ConversationChain_Chains implements INode {
             }
         }
 
-        const chain = prepareChain(nodeData, this.sessionId, options.chatHistory)
-
         const loggerHandler = new ConsoleCallbackHandler(options.logger)
         const additionalCallback = await additionalCallbacks(nodeData, options)
 
@@ -143,20 +145,18 @@ class ConversationChain_Chains implements INode {
     }
 }
 
-const prepareChatPrompt = (nodeData: INodeData) => {
+const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => {
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const prompt = nodeData.inputs?.systemMessagePrompt as string
     const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate
 
     if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) {
         const sysPrompt = chatPromptTemplate.promptMessages[0]
         const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1]
-        const chatPrompt = ChatPromptTemplate.fromMessages([
-            sysPrompt,
-            new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
-            humanPrompt
-        ])
+        const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt]
+        if (humanImageMessages.length) messages.push(...humanImageMessages)
 
+        const chatPrompt = ChatPromptTemplate.fromMessages(messages)
         if ((chatPromptTemplate as any).promptValues) {
             // @ts-ignore
             chatPrompt.promptValues = (chatPromptTemplate as any).promptValues
@@ -165,21 +165,46 @@ const prepareChatPrompt = (nodeData: INodeData) => {
         return chatPrompt
     }
 
-    const chatPrompt = ChatPromptTemplate.fromMessages([
+    const messages = [
         SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage),
         new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'),
         HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`)
-    ])
+    ]
+    if (humanImageMessages.length) messages.push(...(humanImageMessages as any[]))
+
+    const chatPrompt = ChatPromptTemplate.fromMessages(messages)
 
     return chatPrompt
 }
 
-const prepareChain = (nodeData: INodeData, sessionId?: string, chatHistory: IMessage[] = []) => {
-    const model = nodeData.inputs?.model as BaseChatModel
+const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
+    const chatHistory = options.chatHistory
+    let model = nodeData.inputs?.model as ChatOpenAI
     const memory = nodeData.inputs?.memory as FlowiseMemory
     const memoryKey = memory.memoryKey ?? 'chat_history'
 
-    const chatPrompt = prepareChatPrompt(nodeData)
+    let humanImageMessages: HumanMessage[] = []
+    if (model instanceof ChatOpenAI) {
+        const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
+
+        if (messageContent?.length) {
+            // Change model to gpt-4-vision
+            model.modelName = 'gpt-4-vision-preview'
+
+            // Change default max token to higher when using gpt-4-vision
+            model.maxTokens = 1024
+
+            for (const msg of messageContent) {
+                humanImageMessages.push(new HumanMessage({ content: [msg] }))
+            }
+        } else {
+            // revert to previous values if image upload is empty
+            model.modelName = model.configuredModel
+            model.maxTokens = model.configuredMaxToken
+        }
+    }
+
+    const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages)
     let promptVariables = {}
     const promptValuesRaw = (chatPrompt as any).promptValues
     if (promptValuesRaw) {
@@ -203,7 +228,7 @@ const prepareChain = (nodeData: INodeData, sessionId?: string, chatHistory: IMes
             },
             ...promptVariables
         },
-        chatPrompt,
+        prepareChatPrompt(nodeData, humanImageMessages),
         model,
         new StringOutputParser()
     ])