feature/Migration

package.json

@@ -22,7 +22,8 @@
         "lint": "eslint \"**/*.{js,jsx,ts,tsx,json,md}\"",
         "lint-fix": "yarn lint --fix",
         "quick": "pretty-quick --staged",
-        "postinstall": "husky install"
+        "postinstall": "husky install",
+        "migration:create": "yarn typeorm migration:create"
     },
     "lint-staged": {
         "*.{js,jsx,ts,tsx,json,md}": "eslint --fix"

packages/server/src/DataSource.ts

@@ -1,26 +1,26 @@
 import 'reflect-metadata'
 import path from 'path'
 import { DataSource } from 'typeorm'
-import { ChatFlow } from './entity/ChatFlow'
-import { ChatMessage } from './entity/ChatMessage'
-import { Credential } from './entity/Credential'
-import { Tool } from './entity/Tool'
 import { getUserHome } from './utils'
+import { entities } from './database/entities'
+import { sqliteMigrations } from './database/migrations/sqlite'
+import { mysqlMigrations } from './database/migrations/mysql'
+import { postgresMigrations } from './database/migrations/postgres'
 
 let appDataSource: DataSource
 
 export const init = async (): Promise<void> => {
     let homePath
-    const synchronize = process.env.OVERRIDE_DATABASE === 'false' ? false : true
     switch (process.env.DATABASE_TYPE) {
         case 'sqlite':
             homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
             appDataSource = new DataSource({
                 type: 'sqlite',
                 database: path.resolve(homePath, 'database.sqlite'),
-                synchronize,
-                entities: [ChatFlow, ChatMessage, Tool, Credential],
-                migrations: []
+                synchronize: false,
+                migrationsRun: false,
+                entities: Object.values(entities),
+                migrations: sqliteMigrations
             })
             break
         case 'mysql':
@@ -32,9 +32,10 @@ export const init = async (): Promise<void> => {
                 password: process.env.DATABASE_PASSWORD,
                 database: process.env.DATABASE_NAME,
                 charset: 'utf8mb4',
-                synchronize,
-                entities: [ChatFlow, ChatMessage, Tool, Credential],
-                migrations: []
+                synchronize: false,
+                migrationsRun: false,
+                entities: Object.values(entities),
+                migrations: mysqlMigrations
             })
             break
         case 'postgres':
@@ -45,19 +46,21 @@ export const init = async (): Promise<void> => {
                 username: process.env.DATABASE_USER,
                 password: process.env.DATABASE_PASSWORD,
                 database: process.env.DATABASE_NAME,
-                synchronize,
-                entities: [ChatFlow, ChatMessage, Tool, Credential],
-                migrations: []
+                synchronize: false,
+                migrationsRun: false,
+                entities: Object.values(entities),
+                migrations: postgresMigrations
             })
             break
         default:
             homePath = process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise')
             appDataSource = new DataSource({
                 type: 'sqlite',
                 database: path.resolve(homePath, 'database.sqlite'),
-                synchronize,
-                entities: [ChatFlow, ChatMessage, Tool, Credential],
-                migrations: []
+                synchronize: false,
+                migrationsRun: false,
+                entities: Object.values(entities),
+                migrations: sqliteMigrations
             })
             break
     }

packages/server/src/entity/ChatFlow.ts → packages/server/src/database/entities/ChatFlow.ts

@@ -1,6 +1,6 @@
 /* eslint-disable */
 import { Entity, Column, CreateDateColumn, UpdateDateColumn, PrimaryGeneratedColumn } from 'typeorm'
-import { IChatFlow } from '../Interface'
+import { IChatFlow } from '../../Interface'
 
 @Entity()
 export class ChatFlow implements IChatFlow {
@@ -22,7 +22,7 @@ export class ChatFlow implements IChatFlow {
     @Column({ nullable: true })
     apikeyid?: string
 
-    @Column({ nullable: true })
+    @Column({ nullable: true, type: 'text' })
     chatbotConfig?: string
 
     @CreateDateColumn()

packages/server/src/entity/ChatMessage.ts → packages/server/src/database/entities/ChatMessage.ts

@@ -1,6 +1,6 @@
 /* eslint-disable */
 import { Entity, Column, CreateDateColumn, PrimaryGeneratedColumn, Index } from 'typeorm'
-import { IChatMessage, MessageType } from '../Interface'
+import { IChatMessage, MessageType } from '../../Interface'
 
 @Entity()
 export class ChatMessage implements IChatMessage {
@@ -17,7 +17,7 @@ export class ChatMessage implements IChatMessage {
     @Column({ type: 'text' })
     content: string
 
-    @Column({ nullable: true })
+    @Column({ nullable: true, type: 'text' })
     sourceDocuments?: string
 
     @CreateDateColumn()

packages/server/src/entity/Credential.ts → packages/server/src/database/entities/Credential.ts

@@ -1,6 +1,6 @@
 /* eslint-disable */
 import { Entity, Column, PrimaryGeneratedColumn, Index, CreateDateColumn, UpdateDateColumn } from 'typeorm'
-import { ICredential } from '../Interface'
+import { ICredential } from '../../Interface'
 
 @Entity()
 export class Credential implements ICredential {
@@ -13,7 +13,7 @@ export class Credential implements ICredential {
     @Column()
     credentialName: string
 
-    @Column()
+    @Column({ type: 'text' })
     encryptedData: string
 
     @CreateDateColumn()

packages/server/src/entity/Tool.ts → packages/server/src/database/entities/Tool.ts

@@ -1,6 +1,6 @@
 /* eslint-disable */
 import { Entity, Column, CreateDateColumn, UpdateDateColumn, PrimaryGeneratedColumn } from 'typeorm'
-import { ITool } from '../Interface'
+import { ITool } from '../../Interface'
 
 @Entity()
 export class Tool implements ITool {
@@ -19,10 +19,10 @@ export class Tool implements ITool {
     @Column({ nullable: true })
     iconSrc?: string
 
-    @Column({ nullable: true })
+    @Column({ nullable: true, type: 'text' })
     schema?: string
 
-    @Column({ nullable: true })
+    @Column({ nullable: true, type: 'text' })
     func?: string
 
     @CreateDateColumn()

packages/server/src/database/entities/index.ts

@@ -0,0 +1,11 @@
+import { ChatFlow } from './ChatFlow'
+import { ChatMessage } from './ChatMessage'
+import { Credential } from './Credential'
+import { Tool } from './Tool'
+
+export const entities = {
+    ChatFlow,
+    ChatMessage,
+    Credential,
+    Tool
+}

packages/server/src/database/migrations/mysql/1693840429259-Init.ts

@@ -0,0 +1,64 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class Init1693840429259 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`chat_flow\` (
+                \`id\` varchar(36) NOT NULL,
+                \`name\` varchar(255) NOT NULL,
+                \`flowData\` text NOT NULL,
+                \`deployed\` tinyint DEFAULT NULL,
+                \`isPublic\` tinyint DEFAULT NULL,
+                \`apikeyid\` varchar(255) DEFAULT NULL,
+                \`chatbotConfig\` varchar(255) DEFAULT NULL,
+                \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`chat_message\` (
+                \`id\` varchar(36) NOT NULL,
+                \`role\` varchar(255) NOT NULL,
+                \`chatflowid\` varchar(255) NOT NULL,
+                \`content\` text NOT NULL,
+                \`sourceDocuments\` varchar(255) DEFAULT NULL,
+                \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`),
+                KEY \`IDX_e574527322272fd838f4f0f3d3\` (\`chatflowid\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`credential\` (
+                \`id\` varchar(36) NOT NULL,
+                \`name\` varchar(255) NOT NULL,
+                \`credentialName\` varchar(255) NOT NULL,
+                \`encryptedData\` varchar(255) NOT NULL,
+                \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`tool\` (
+                \`id\` varchar(36) NOT NULL,
+                \`name\` varchar(255) NOT NULL,
+                \`description\` text NOT NULL,
+                \`color\` varchar(255) NOT NULL,
+                \`iconSrc\` varchar(255) DEFAULT NULL,
+                \`schema\` varchar(255) DEFAULT NULL,
+                \`func\` varchar(255) DEFAULT NULL,
+                \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE chat_flow`)
+        await queryRunner.query(`DROP TABLE chat_message`)
+        await queryRunner.query(`DROP TABLE credential`)
+        await queryRunner.query(`DROP TABLE tool`)
+    }
+}

packages/server/src/database/migrations/mysql/1693997791471-ModifyChatFlow.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class ModifyChatFlow1693997791471 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_flow\` MODIFY \`chatbotConfig\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_flow\` MODIFY \`chatbotConfig\` VARCHAR;`)
+    }
+}

packages/server/src/database/migrations/mysql/1693999022236-ModifyChatMessage.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class ModifyChatMessage1693999022236 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_message\` MODIFY \`sourceDocuments\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_message\` MODIFY \`sourceDocuments\` VARCHAR;`)
+    }
+}

packages/server/src/database/migrations/mysql/1693999261583-ModifyCredential.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class ModifyCredential1693999261583 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`credential\` MODIFY \`encryptedData\` TEXT NOT NULL;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`credential\` MODIFY \`encryptedData\` VARCHAR NOT NULL;`)
+    }
+}

packages/server/src/database/migrations/mysql/1694001465232-ModifyTool.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class ModifyTool1694001465232 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`tool\` MODIFY \`schema\` TEXT, MODIFY \`func\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`tool\` MODIFY \`schema\` VARCHAR, MODIFY \`func\` VARCHAR;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -0,0 +1,13 @@
+import { Init1693840429259 } from './1693840429259-Init'
+import { ModifyChatFlow1693997791471 } from './1693997791471-ModifyChatFlow'
+import { ModifyChatMessage1693999022236 } from './1693999022236-ModifyChatMessage'
+import { ModifyCredential1693999261583 } from './1693999261583-ModifyCredential'
+import { ModifyTool1694001465232 } from './1694001465232-ModifyTool'
+
+export const mysqlMigrations = [
+    Init1693840429259,
+    ModifyChatFlow1693997791471,
+    ModifyChatMessage1693999022236,
+    ModifyCredential1693999261583,
+    ModifyTool1694001465232
+]

packages/server/src/database/migrations/postgres/1693891895163-Init.ts

@@ -0,0 +1,64 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class Init1693891895163 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS chat_flow (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "name" varchar NOT NULL,
+                "flowData" text NOT NULL,
+                deployed bool NULL,
+                "isPublic" bool NULL,
+                apikeyid varchar NULL,
+                "chatbotConfig" varchar NULL,
+                "createdDate" timestamp NOT NULL DEFAULT now(),
+                "updatedDate" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_3c7cea7d047ac4b91764574cdbf" PRIMARY KEY (id)
+            );`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS chat_message (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "role" varchar NOT NULL,
+                chatflowid varchar NOT NULL,
+                "content" text NOT NULL,
+                "sourceDocuments" varchar NULL,
+                "createdDate" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_3cc0d85193aade457d3077dd06b" PRIMARY KEY (id)
+            );`
+        )
+        await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_e574527322272fd838f4f0f3d3" ON chat_message USING btree (chatflowid);`)
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS credential (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "name" varchar NOT NULL,
+                "credentialName" varchar NOT NULL,
+                "encryptedData" varchar NOT NULL,
+                "createdDate" timestamp NOT NULL DEFAULT now(),
+                "updatedDate" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_3a5169bcd3d5463cefeec78be82" PRIMARY KEY (id)
+            );`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS tool (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "name" varchar NOT NULL,
+                description text NOT NULL,
+                color varchar NOT NULL,
+                "iconSrc" varchar NULL,
+                "schema" varchar NULL,
+                func varchar NULL,
+                "createdDate" timestamp NOT NULL DEFAULT now(),
+                "updatedDate" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_3bf5b1016a384916073184f99b7" PRIMARY KEY (id)
+            );`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE chat_flow`)
+        await queryRunner.query(`DROP TABLE chat_message`)
+        await queryRunner.query(`DROP TABLE credential`)
+        await queryRunner.query(`DROP TABLE tool`)
+    }
+}

packages/server/src/database/migrations/postgres/1693995626941-ModifyChatFlow.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class ModifyChatFlow1693995626941 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" ALTER COLUMN "chatbotConfig" TYPE TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" ALTER COLUMN "chatbotConfig" TYPE VARCHAR;`)
+    }
+}

packages/server/src/database/migrations/postgres/1693996694528-ModifyChatMessage.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class ModifyChatMessage1693996694528 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "sourceDocuments" TYPE TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "sourceDocuments" TYPE VARCHAR;`)
+    }
+}

packages/server/src/database/migrations/postgres/1693997070000-ModifyCredential.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class ModifyCredential1693997070000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "credential" ALTER COLUMN "encryptedData" TYPE TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "credential" ALTER COLUMN "encryptedData" TYPE VARCHAR;`)
+    }
+}