From 2cbf82126fea661b2884f047455bd89cd0e0e1ab Mon Sep 17 00:00:00 2001 From: martinibach Date: Mon, 25 Nov 2024 12:11:35 +0100 Subject: [PATCH 1/7] init voice example integrations to functions --- voice/package.json | 46 ++-- voice/pnpm-lock.yaml | 206 +++++++----------- voice/src/functions/deepgram/index.ts | 2 + voice/src/functions/deepgram/listen.ts | 65 ++++++ voice/src/functions/deepgram/speak.ts | 82 +++++++ voice/src/functions/deepgram/utils/client.ts | 19 ++ voice/src/functions/index.ts | 4 + .../functions/openai/chat/completionsBase.ts | 92 ++++++++ .../openai/chat/completionsStream.ts | 193 ++++++++++++++++ voice/src/functions/openai/chat/index.ts | 2 + voice/src/functions/openai/index.ts | 2 + .../openai/thread/createAssistant.ts | 34 +++ .../openai/thread/createMessageOnThread.ts | 28 +++ .../functions/openai/thread/createThread.ts | 19 ++ voice/src/functions/openai/thread/index.ts | 4 + .../src/functions/openai/thread/runThread.ts | 31 +++ voice/src/functions/openai/types/events.ts | 17 ++ voice/src/functions/openai/types/index.ts | 1 + .../functions/openai/utils/aggregateStream.ts | 26 +++ voice/src/functions/openai/utils/client.ts | 21 ++ voice/src/functions/openai/utils/cost.ts | 24 ++ voice/src/functions/openai/utils/index.ts | 4 + .../functions/openai/utils/mergeToolCalls.ts | 31 +++ voice/src/functions/twilio/call.ts | 34 +++ voice/src/functions/twilio/index.ts | 1 + voice/src/functions/twilio/utils/client.ts | 23 ++ voice/src/functions/websocket/index.ts | 2 + voice/src/functions/websocket/listen.ts | 64 ++++++ voice/src/functions/websocket/send.ts | 27 +++ voice/src/functions/websocket/types/events.ts | 12 + voice/src/functions/websocket/types/index.ts | 1 + voice/src/functions/websocket/utils/client.ts | 31 +++ voice/src/services.ts | 43 +++- .../workflows/conversation/conversation.ts | 16 +- voice/src/workflows/room/events.ts | 2 +- voice/src/workflows/room/room.ts | 44 ++-- voice/src/workflows/twilioCall.ts | 7 +- 37 files changed, 1060 insertions(+), 200 deletions(-) create mode 100644 voice/src/functions/deepgram/index.ts create mode 100644 voice/src/functions/deepgram/listen.ts create mode 100644 voice/src/functions/deepgram/speak.ts create mode 100644 voice/src/functions/deepgram/utils/client.ts create mode 100644 voice/src/functions/openai/chat/completionsBase.ts create mode 100644 voice/src/functions/openai/chat/completionsStream.ts create mode 100644 voice/src/functions/openai/chat/index.ts create mode 100644 voice/src/functions/openai/index.ts create mode 100644 voice/src/functions/openai/thread/createAssistant.ts create mode 100644 voice/src/functions/openai/thread/createMessageOnThread.ts create mode 100644 voice/src/functions/openai/thread/createThread.ts create mode 100644 voice/src/functions/openai/thread/index.ts create mode 100644 voice/src/functions/openai/thread/runThread.ts create mode 100644 voice/src/functions/openai/types/events.ts create mode 100644 voice/src/functions/openai/types/index.ts create mode 100644 voice/src/functions/openai/utils/aggregateStream.ts create mode 100644 voice/src/functions/openai/utils/client.ts create mode 100644 voice/src/functions/openai/utils/cost.ts create mode 100644 voice/src/functions/openai/utils/index.ts create mode 100644 voice/src/functions/openai/utils/mergeToolCalls.ts create mode 100644 voice/src/functions/twilio/call.ts create mode 100644 voice/src/functions/twilio/index.ts create mode 100644 voice/src/functions/twilio/utils/client.ts create mode 100644 voice/src/functions/websocket/index.ts create mode 100644 voice/src/functions/websocket/listen.ts create mode 100644 voice/src/functions/websocket/send.ts create mode 100644 voice/src/functions/websocket/types/events.ts create mode 100644 voice/src/functions/websocket/types/index.ts create mode 100644 voice/src/functions/websocket/utils/client.ts diff --git a/voice/package.json b/voice/package.json index ced09a4..01a6a8b 100644 --- a/voice/package.json +++ b/voice/package.json @@ -17,35 +17,31 @@ "author": "", "license": "ISC", "dependencies": { - "@deepgram/sdk": "^3.6.0", - "@restackio/ai": "^0.0.85", - "@restackio/integrations-deepgram": "^0.0.13", - "@restackio/integrations-openai": "^0.0.34", - "@restackio/integrations-twilio": "^0.0.10", - "@restackio/integrations-websocket": "^0.0.17", + "@restackio/ai": "0.0.85", "@temporalio/workflow": "1.11.1", - "cors": "^2.8.5", - "dotenv": "^16.4.5", - "express": "^4.21.0", - "openai": "^4.59.0", - "ts-node": "^10.9.2", - "twilio": "^5.3.0", - "typescript": "^5.6.2", - "uuid": "^10.0.0", - "ws": "^8.18.0", - "zod": "^3.23.8" + "cors": "2.8.5", + "dotenv": "16.4.5", + "express": "4.21.0", + "openai": "4.59.0", + "ts-node": "10.9.2", + "twilio": "5.3.0", + "@deepgram/sdk": "3.6.0", + "typescript": "5.6.2", + "uuid": "10.0.0", + "ws": "8.18.0", + "zod": "3.23.8" }, "devDependencies": { - "@restackio/cloud": "^1.0.19", - "@types/cors": "^2.8.17", - "@types/express": "^4.17.21", - "@types/node": "^22.5.4", - "@types/uuid": "^10.0.0", - "@types/ws": "^8.5.12", - "nodemon": "^3.1.4", - "ts-node-dev": "^2.0.0" + "@restackio/cloud": "1.0.19", + "@types/cors": "2.8.17", + "@types/express": "4.17.21", + "@types/node": "22.5.4", + "@types/uuid": "10.0.0", + "@types/ws": "8.5.12", + "nodemon": "3.1.4", + "ts-node-dev": "2.0.0" }, "optionalDependencies": { - "bufferutil": "^4.0.8" + "bufferutil": "4.0.8" } } diff --git a/voice/pnpm-lock.yaml b/voice/pnpm-lock.yaml index eb1634c..59de3ad 100644 --- a/voice/pnpm-lock.yaml +++ b/voice/pnpm-lock.yaml @@ -9,85 +9,73 @@ importers: .: dependencies: '@deepgram/sdk': - specifier: ^3.6.0 - version: 3.9.0(bufferutil@4.0.8) + specifier: 3.6.0 + version: 3.6.0(bufferutil@4.0.8) '@restackio/ai': - specifier: ^0.0.85 + specifier: 0.0.85 version: 0.0.85 - '@restackio/integrations-deepgram': - specifier: ^0.0.13 - version: 0.0.13(bufferutil@4.0.8) - '@restackio/integrations-openai': - specifier: ^0.0.34 - version: 0.0.34(zod@3.23.8) - '@restackio/integrations-twilio': - specifier: ^0.0.10 - version: 0.0.10 - '@restackio/integrations-websocket': - specifier: ^0.0.17 - version: 0.0.17(bufferutil@4.0.8) '@temporalio/workflow': specifier: 1.11.1 version: 1.11.1 cors: - specifier: ^2.8.5 + specifier: 2.8.5 version: 2.8.5 dotenv: - specifier: ^16.4.5 + specifier: 16.4.5 version: 16.4.5 express: - specifier: ^4.21.0 - version: 4.21.1 + specifier: 4.21.0 + version: 4.21.0 openai: - specifier: ^4.59.0 - version: 4.73.0(zod@3.23.8) + specifier: 4.59.0 + version: 4.59.0(zod@3.23.8) ts-node: - specifier: ^10.9.2 - version: 10.9.2(@swc/core@1.9.3)(@types/node@22.9.1)(typescript@5.6.3) + specifier: 10.9.2 + version: 10.9.2(@swc/core@1.9.3)(@types/node@22.5.4)(typescript@5.6.2) twilio: - specifier: ^5.3.0 - version: 5.3.6 + specifier: 5.3.0 + version: 5.3.0 typescript: - specifier: ^5.6.2 - version: 5.6.3 + specifier: 5.6.2 + version: 5.6.2 uuid: - specifier: ^10.0.0 + specifier: 10.0.0 version: 10.0.0 ws: - specifier: ^8.18.0 + specifier: 8.18.0 version: 8.18.0(bufferutil@4.0.8) zod: - specifier: ^3.23.8 + specifier: 3.23.8 version: 3.23.8 optionalDependencies: bufferutil: - specifier: ^4.0.8 + specifier: 4.0.8 version: 4.0.8 devDependencies: '@restackio/cloud': - specifier: ^1.0.19 + specifier: 1.0.19 version: 1.0.19 '@types/cors': - specifier: ^2.8.17 + specifier: 2.8.17 version: 2.8.17 '@types/express': - specifier: ^4.17.21 + specifier: 4.17.21 version: 4.17.21 '@types/node': - specifier: ^22.5.4 - version: 22.9.1 + specifier: 22.5.4 + version: 22.5.4 '@types/uuid': - specifier: ^10.0.0 + specifier: 10.0.0 version: 10.0.0 '@types/ws': - specifier: ^8.5.12 - version: 8.5.13 + specifier: 8.5.12 + version: 8.5.12 nodemon: - specifier: ^3.1.4 - version: 3.1.7 + specifier: 3.1.4 + version: 3.1.4 ts-node-dev: - specifier: ^2.0.0 - version: 2.0.0(@swc/core@1.9.3)(@types/node@22.9.1)(typescript@5.6.3) + specifier: 2.0.0 + version: 2.0.0(@swc/core@1.9.3)(@types/node@22.5.4)(typescript@5.6.2) packages: @@ -99,8 +87,8 @@ packages: resolution: {integrity: sha512-8B1C/oTxTxyHlSFubAhNRgCbQ2SQ5wwvtlByn8sDYZvdDtdn/VE2yEPZ4BvUnrKWmsbTQY6/ooLV+9Ka2qmDSQ==} engines: {node: '>=18.0.0'} - '@deepgram/sdk@3.9.0': - resolution: {integrity: sha512-X/7JzoYjCObyEaPb2Dgnkwk2LwRe4bw0FJJCLdkjpnFfJCFgA9IWgRD8FEUI6/hp8dW/CqqXkGPA2Q3DIsVG8A==} + '@deepgram/sdk@3.6.0': + resolution: {integrity: sha512-uErkoOyBjdk3u4UyowxN6LtjuLv8s1V/29WVeR2Y2M1wKGo93s8dlYFsF3R1SWeMaeJOimpCcwjWvMQbYrmS6Q==} engines: {node: '>=18.0.0'} '@grpc/grpc-js@1.12.2': @@ -201,18 +189,6 @@ packages: resolution: {integrity: sha512-UJNfM4di6qGa3EqAZzUdcrGjK6tIj2YmQbGZouBZjpHDqG3kbzMZkgNPcW/UsCp0HYCKS36pUmDy6m1GkMFgNQ==} engines: {node: '>=18'} - '@restackio/integrations-deepgram@0.0.13': - resolution: {integrity: sha512-sg4dVbUo8Oba2Bvhc0GaOcJjA8k4l4hVoEMT23un4XYiusLGsGKEpbvAjB4lu9ABM+TMLHVGh28YPdy0x4T27w==} - - '@restackio/integrations-openai@0.0.34': - resolution: {integrity: sha512-lXI9nHCDHZmSV52CFqVxpbO0egEQC4daLI9u+P8OxrUcCAfsPfgItO35hHi+orUubUGcQJAN4NdB3bTr2xqqeQ==} - - '@restackio/integrations-twilio@0.0.10': - resolution: {integrity: sha512-Yx+vVBy31bKnttoq2rcyWrmSTMzi5wJCXKUEpyxCyhF4I66nbjuX+acCprhayVt4Gca0euj/mY7Acy6hMpVlCg==} - - '@restackio/integrations-websocket@0.0.17': - resolution: {integrity: sha512-JLPW1rU+BAGHAw4GIw2HZtHgRbzaWzLZVpHnqkgU99M5mEUpu4bCT5sLHnwIc8tpWKm/vQZR0hbsFjTtOVBziw==} - '@swc/core-darwin-arm64@1.9.3': resolution: {integrity: sha512-hGfl/KTic/QY4tB9DkTbNuxy5cV4IeejpPD4zo+Lzt4iLlDWIeANL4Fkg67FiVceNJboqg48CUX+APhDHO5G1w==} engines: {node: '>=10'} @@ -370,8 +346,8 @@ packages: '@types/node@18.19.64': resolution: {integrity: sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==} - '@types/node@22.9.1': - resolution: {integrity: sha512-p8Yy/8sw1caA8CdRIQBG5tiLHmxtQKObCijiAa9Ez+d4+PRffM4054xbju0msf+cvhJpnFEeNjxmVT/0ipktrg==} + '@types/node@22.5.4': + resolution: {integrity: sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg==} '@types/qs@6.9.17': resolution: {integrity: sha512-rX4/bPcfmvxHDv0XjfJELTTr+iB+tn032nPILqHm5wbthUUUuVtNGGqzhya9XUxjTP8Fpr0qYgSZZKxGY++svQ==} @@ -394,8 +370,8 @@ packages: '@types/uuid@10.0.0': resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} - '@types/ws@8.5.13': - resolution: {integrity: sha512-osM/gWBTPKgHV8XkTunnegTRIsvF6owmf5w+JtAfOw472dptdm0dlGv4xCt6GwQRcC2XVOvvRE/0bAoQcL2QkA==} + '@types/ws@8.5.12': + resolution: {integrity: sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==} '@webassemblyjs/ast@1.14.1': resolution: {integrity: sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==} @@ -604,8 +580,8 @@ packages: cookie-signature@1.0.6: resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} - cookie@0.7.1: - resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==} + cookie@0.6.0: + resolution: {integrity: sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==} engines: {node: '>= 0.6'} cors@2.8.5: @@ -751,8 +727,8 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} - express@4.21.1: - resolution: {integrity: sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==} + express@4.21.0: + resolution: {integrity: sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==} engines: {node: '>= 0.10.0'} fast-deep-equal@3.1.3: @@ -1075,8 +1051,8 @@ packages: node-releases@2.0.18: resolution: {integrity: sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==} - nodemon@3.1.7: - resolution: {integrity: sha512-hLj7fuMow6f0lbB0cD14Lz2xNjwsyruH251Pk4t/yIitCFJbmY1myuLlHm/q06aST4jg6EgAh74PIBBrRqpVAQ==} + nodemon@3.1.4: + resolution: {integrity: sha512-wjPBbFhtpJwmIeY2yP7QF+UKzPfltVGtfce1g/bB15/8vCGZj8uxD62b/b9M9/WVgme0NZudpownKN+c0plXlQ==} engines: {node: '>=10'} hasBin: true @@ -1103,8 +1079,8 @@ packages: resolution: {integrity: sha512-mnkeQ1qP5Ue2wd+aivTD3NHd/lZ96Lu0jgf0pwktLPtx6cTZiH7tyeGRRHs0zX0rbrahXPnXlUnbeXyaBBuIaw==} engines: {node: '>=18'} - openai@4.73.0: - resolution: {integrity: sha512-NZstV77w3CEol9KQTRBRQ15+Sw6nxVTicAULSjYO4wn9E5gw72Mtp3fAVaBFXyyVPws4241YmFG6ya4L8v03tA==} + openai@4.59.0: + resolution: {integrity: sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg==} hasBin: true peerDependencies: zod: ^3.23.8 @@ -1404,8 +1380,8 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - twilio@5.3.6: - resolution: {integrity: sha512-izHce9sWpiIYyFeZ5pJb5KQeHQ6NDyGuCQ+BOTbBS64ZWq+0InWXvWjZsXbFwGFrhn5MQq0ulouLtYOXiEYY8g==} + twilio@5.3.0: + resolution: {integrity: sha512-bwveAxChPPFR2umttraRjUJdq/WY0OJCCgetzuKqLoGYqYSyYGsiFYYFAB5EOL/XnzCQNwAvq5622u+jqMTLOA==} engines: {node: '>=14.0'} type-fest@2.19.0: @@ -1416,8 +1392,8 @@ packages: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} - typescript@5.6.3: - resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==} + typescript@5.6.2: + resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==} engines: {node: '>=14.17'} hasBin: true @@ -1557,10 +1533,9 @@ snapshots: dependencies: dayjs: 1.11.13 - '@deepgram/sdk@3.9.0(bufferutil@4.0.8)': + '@deepgram/sdk@3.6.0(bufferutil@4.0.8)': dependencies: '@deepgram/captions': 1.2.0 - '@types/node': 18.19.64 cross-fetch: 3.1.8 deepmerge: 4.3.1 events: 3.3.0 @@ -1681,35 +1656,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@restackio/integrations-deepgram@0.0.13(bufferutil@4.0.8)': - dependencies: - '@deepgram/sdk': 3.9.0(bufferutil@4.0.8) - transitivePeerDependencies: - - bufferutil - - encoding - - utf-8-validate - - '@restackio/integrations-openai@0.0.34(zod@3.23.8)': - dependencies: - openai: 4.73.0(zod@3.23.8) - transitivePeerDependencies: - - encoding - - zod - - '@restackio/integrations-twilio@0.0.10': - dependencies: - twilio: 5.3.6 - transitivePeerDependencies: - - debug - - supports-color - - '@restackio/integrations-websocket@0.0.17(bufferutil@4.0.8)': - dependencies: - ws: 8.18.0(bufferutil@4.0.8) - transitivePeerDependencies: - - bufferutil - - utf-8-validate - '@swc/core-darwin-arm64@1.9.3': optional: true @@ -1853,15 +1799,15 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.38 - '@types/node': 22.9.1 + '@types/node': 22.5.4 '@types/connect@3.4.38': dependencies: - '@types/node': 22.9.1 + '@types/node': 22.5.4 '@types/cors@2.8.17': dependencies: - '@types/node': 22.9.1 + '@types/node': 22.5.4 '@types/eslint-scope@3.7.7': dependencies: @@ -1877,7 +1823,7 @@ snapshots: '@types/express-serve-static-core@4.19.6': dependencies: - '@types/node': 22.9.1 + '@types/node': 22.5.4 '@types/qs': 6.9.17 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -1897,14 +1843,14 @@ snapshots: '@types/node-fetch@2.6.12': dependencies: - '@types/node': 22.9.1 + '@types/node': 22.5.4 form-data: 4.0.1 '@types/node@18.19.64': dependencies: undici-types: 5.26.5 - '@types/node@22.9.1': + '@types/node@22.5.4': dependencies: undici-types: 6.19.8 @@ -1915,12 +1861,12 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 22.9.1 + '@types/node': 22.5.4 '@types/serve-static@1.15.7': dependencies: '@types/http-errors': 2.0.4 - '@types/node': 22.9.1 + '@types/node': 22.5.4 '@types/send': 0.17.4 '@types/strip-bom@3.0.0': {} @@ -1929,9 +1875,9 @@ snapshots: '@types/uuid@10.0.0': {} - '@types/ws@8.5.13': + '@types/ws@8.5.12': dependencies: - '@types/node': 22.9.1 + '@types/node': 22.5.4 '@webassemblyjs/ast@1.14.1': dependencies: @@ -2187,7 +2133,7 @@ snapshots: cookie-signature@1.0.6: {} - cookie@0.7.1: {} + cookie@0.6.0: {} cors@2.8.5: dependencies: @@ -2295,14 +2241,14 @@ snapshots: events@3.3.0: {} - express@4.21.1: + express@4.21.0: dependencies: accepts: 1.3.8 array-flatten: 1.1.1 body-parser: 1.20.3 content-disposition: 0.5.4 content-type: 1.0.5 - cookie: 0.7.1 + cookie: 0.6.0 cookie-signature: 1.0.6 debug: 2.6.9 depd: 2.0.0 @@ -2500,7 +2446,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 22.9.1 + '@types/node': 22.5.4 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -2606,7 +2552,7 @@ snapshots: node-releases@2.0.18: {} - nodemon@3.1.7: + nodemon@3.1.4: dependencies: chokidar: 3.6.0 debug: 4.3.7(supports-color@5.5.0) @@ -2640,15 +2586,17 @@ snapshots: is-inside-container: 1.0.0 is-wsl: 3.1.0 - openai@4.73.0(zod@3.23.8): + openai@4.59.0(zod@3.23.8): dependencies: '@types/node': 18.19.64 '@types/node-fetch': 2.6.12 + '@types/qs': 6.9.17 abort-controller: 3.0.0 agentkeepalive: 4.5.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 node-fetch: 2.7.0 + qs: 6.13.1 optionalDependencies: zod: 3.23.8 transitivePeerDependencies: @@ -2684,7 +2632,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 22.9.1 + '@types/node': 22.5.4 long: 5.2.3 proxy-addr@2.0.7: @@ -2913,7 +2861,7 @@ snapshots: tree-kill@1.2.2: {} - ts-node-dev@2.0.0(@swc/core@1.9.3)(@types/node@22.9.1)(typescript@5.6.3): + ts-node-dev@2.0.0(@swc/core@1.9.3)(@types/node@22.5.4)(typescript@5.6.2): dependencies: chokidar: 3.6.0 dynamic-dedupe: 0.3.0 @@ -2923,29 +2871,29 @@ snapshots: rimraf: 2.7.1 source-map-support: 0.5.21 tree-kill: 1.2.2 - ts-node: 10.9.2(@swc/core@1.9.3)(@types/node@22.9.1)(typescript@5.6.3) + ts-node: 10.9.2(@swc/core@1.9.3)(@types/node@22.5.4)(typescript@5.6.2) tsconfig: 7.0.0 - typescript: 5.6.3 + typescript: 5.6.2 transitivePeerDependencies: - '@swc/core' - '@swc/wasm' - '@types/node' - ts-node@10.9.2(@swc/core@1.9.3)(@types/node@22.9.1)(typescript@5.6.3): + ts-node@10.9.2(@swc/core@1.9.3)(@types/node@22.5.4)(typescript@5.6.2): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 22.9.1 + '@types/node': 22.5.4 acorn: 8.14.0 acorn-walk: 8.3.4 arg: 4.1.3 create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.6.3 + typescript: 5.6.2 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 optionalDependencies: @@ -2960,7 +2908,7 @@ snapshots: tslib@2.8.1: {} - twilio@5.3.6: + twilio@5.3.0: dependencies: axios: 1.7.7 dayjs: 1.11.13 @@ -2980,7 +2928,7 @@ snapshots: media-typer: 0.3.0 mime-types: 2.1.35 - typescript@5.6.3: {} + typescript@5.6.2: {} undefsafe@2.0.5: {} diff --git a/voice/src/functions/deepgram/index.ts b/voice/src/functions/deepgram/index.ts new file mode 100644 index 0000000..3f8dba7 --- /dev/null +++ b/voice/src/functions/deepgram/index.ts @@ -0,0 +1,2 @@ +export * from "./listen"; +export * from "./speak"; diff --git a/voice/src/functions/deepgram/listen.ts b/voice/src/functions/deepgram/listen.ts new file mode 100644 index 0000000..341e5bc --- /dev/null +++ b/voice/src/functions/deepgram/listen.ts @@ -0,0 +1,65 @@ +import { FunctionFailure, log } from "@restackio/ai/function"; +import { Buffer } from "node:buffer"; +import { deepgramClient } from "./utils/client"; +import { PrerecordedSchema } from "@deepgram/sdk"; + +export async function deepgramListen({ + base64Payload, + options = { + model: "nova-2", + punctuate: true, + interim_results: true, + endpointing: 500, + utterance_end_ms: 2000, + }, + twilioEncoding, + apiKey, +}: { + base64Payload: string; + options?: PrerecordedSchema; + twilioEncoding?: boolean; + apiKey?: string; +}) { + if (!base64Payload) { + throw FunctionFailure.nonRetryable("No audio file"); + } + + try { + const decodedBuffer = Buffer.from(base64Payload, "base64"); + + const deepgram = deepgramClient({ apiKey }); + + const response = await deepgram.listen.prerecorded.transcribeFile( + decodedBuffer, + { + ...options, + ...(twilioEncoding && { + encoding: "mulaw", + sample_rate: 8000, + }), + } + ); + + if (response.error) { + log.error("deepgramListen error", { error: response.error }); + } + + const result = response.result; + const firstChannel = result.results?.channels?.[0]; + + const transcript = firstChannel?.alternatives?.[0]?.transcript; + + let language = ""; + if (options.detect_language) { + language = firstChannel?.detected_language; + } + + return { + transcript, + language, + result, + }; + } catch (error) { + throw new Error(`Deepgram TTS error ${error}`); + } +} diff --git a/voice/src/functions/deepgram/speak.ts b/voice/src/functions/deepgram/speak.ts new file mode 100644 index 0000000..1037bf4 --- /dev/null +++ b/voice/src/functions/deepgram/speak.ts @@ -0,0 +1,82 @@ +import { FunctionFailure, log } from "@restackio/ai/function"; +import { Buffer } from "node:buffer"; +import { deepgramClient } from "./utils/client"; +import { SpeakSchema } from "@deepgram/sdk"; + +const getAudioBuffer = async (stream: ReadableStream) => { + const reader = stream.getReader(); + const chunks = []; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + chunks.push(value); + } + + const dataArray = chunks.reduce( + (acc, chunk) => Uint8Array.from([...acc, ...chunk]), + new Uint8Array(0) + ); + + const buffer = Buffer.from(dataArray.buffer); + return buffer; +}; + +export async function deepgramSpeak({ + text, + options = { + model: "aura-arcas-en", + }, + twilioEncoding, + apiKey, +}: { + text: string; + options?: SpeakSchema; + twilioEncoding?: boolean; + apiKey?: string; +}) { + if (!text.length) { + log.error("Text is empty"); + throw FunctionFailure.nonRetryable("Text is empty"); + } + + try { + const deepgram = deepgramClient({ apiKey }); + const response = await deepgram.speak.request( + { text }, + { + ...options, + ...(twilioEncoding && { + encoding: "mulaw", + sample_rate: 8000, + container: "none", + }), + } + ); + const stream = await response.getStream(); + + if (!stream) { + log.error("Deepgram speak stream error", { response }); + throw new Error(`Deepgram speak stream error ${response}`); + } + + const buffer = await getAudioBuffer(stream); + if (!buffer) { + log.error("Deepgram audio buffer error", { stream }); + throw new Error(`Deepgram audio buffer error ${stream}`); + } + const base64String = buffer.toString("base64"); + log.info("deepgramSpeak: ", { + audioLength: base64String.length, + }); + return { + media: { + payload: base64String, + }, + }; + } catch (error) { + log.error("Deepgram TTS error", { error }); + throw new Error(`Deepgram TTS error ${error}`); + } +} diff --git a/voice/src/functions/deepgram/utils/client.ts b/voice/src/functions/deepgram/utils/client.ts new file mode 100644 index 0000000..8564c7a --- /dev/null +++ b/voice/src/functions/deepgram/utils/client.ts @@ -0,0 +1,19 @@ +import { createClient, DeepgramClient } from "@deepgram/sdk"; +import "dotenv/config"; + +let clientDeepgram: DeepgramClient; + +export function deepgramClient({ + apiKey = process.env.DEEPGRAM_API_KEY, +}: { + apiKey: string; +}) { + if (!apiKey) { + throw new Error("API key is required to create Deepgram client."); + } + + if (!clientDeepgram) { + clientDeepgram = createClient(apiKey); + } + return clientDeepgram; +} diff --git a/voice/src/functions/index.ts b/voice/src/functions/index.ts index d3369aa..e52449c 100644 --- a/voice/src/functions/index.ts +++ b/voice/src/functions/index.ts @@ -1,2 +1,6 @@ export * from "./erp"; export * from "./utils"; +export * from "./deepgram"; +export * from "./twilio"; +export * from "./websocket"; +export * from "./openai"; diff --git a/voice/src/functions/openai/chat/completionsBase.ts b/voice/src/functions/openai/chat/completionsBase.ts new file mode 100644 index 0000000..0889059 --- /dev/null +++ b/voice/src/functions/openai/chat/completionsBase.ts @@ -0,0 +1,92 @@ +import { FunctionFailure, log } from "@restackio/ai/function"; +import OpenAI from "openai/index"; +import { ChatCompletionCreateParamsNonStreaming } from "openai/resources/chat/completions"; +import { openaiClient } from "../utils/client"; +import { openaiCost, Price } from "../utils/cost"; +import { ChatCompletion, ChatModel } from "openai/resources/index"; + +export type UsageOutput = { tokens: number; cost: number }; + +export type OpenAIChatInput = { + userContent: string; + systemContent?: string; + model?: ChatModel; + jsonSchema?: { + name: string; + schema: Record; + }; + price?: Price; + apiKey?: string; + params?: ChatCompletionCreateParamsNonStreaming; + tools?: OpenAI.Chat.Completions.ChatCompletionTool[]; + toolChoice?: OpenAI.Chat.Completions.ChatCompletionToolChoiceOption; +}; + +export const openaiChatCompletionsBase = async ({ + userContent, + systemContent = "", + model = "gpt-4o-mini", + jsonSchema, + price, + apiKey, + params, + tools, + toolChoice, +}: OpenAIChatInput): Promise<{ result: ChatCompletion; cost?: number }> => { + try { + const openai = openaiClient({ apiKey }); + + const isO1Model = model.startsWith("o1-"); + + const o1ModelParams = { + temperature: 1, + top_p: 1, + frequency_penalty: 0, + presence_penalty: 0, + }; + + const chatParams: ChatCompletionCreateParamsNonStreaming = { + messages: [ + ...(systemContent ? [{ role: "system" as const, content: systemContent }] : []), + { role: "user" as const, content: userContent }, + ...(params?.messages ?? []), + ], + ...(jsonSchema && { + response_format: { + type: "json_schema", + json_schema: { + name: jsonSchema.name, + strict: true, + schema: jsonSchema.schema, + }, + }, + }), + model, + ...(tools && { tools }), + ...(toolChoice && { tool_choice: toolChoice }), + ...params, + ...(isO1Model && o1ModelParams), + }; + + log.debug("OpenAI chat completion params", { + chatParams, + }); + + const completion = await openai.chat.completions.create(chatParams); + + return { + result: completion, + cost: + price && + openaiCost({ + price, + tokensCount: { + input: completion.usage?.prompt_tokens ?? 0, + output: completion.usage?.completion_tokens ?? 0, + }, + }), + }; + } catch (error) { + throw FunctionFailure.nonRetryable(`Error OpenAI chat: ${error}`); + } +}; \ No newline at end of file diff --git a/voice/src/functions/openai/chat/completionsStream.ts b/voice/src/functions/openai/chat/completionsStream.ts new file mode 100644 index 0000000..bcd9097 --- /dev/null +++ b/voice/src/functions/openai/chat/completionsStream.ts @@ -0,0 +1,193 @@ +import OpenAI from "openai/index"; +import { ChatCompletionChunk } from "openai/resources/chat/completions"; + +import Restack from "@restackio/ai"; +import { currentWorkflow, log } from "@restackio/ai/function"; + +import { StreamEvent, ToolCallEvent } from "../types/events"; + +import { aggregateStreamChunks } from "../utils/aggregateStream"; +import { mergeToolCalls } from "../utils/mergeToolCalls"; +import { openaiClient } from "../utils/client"; +import { openaiCost, Price } from "../utils/cost"; +import { SendWorkflowEvent } from "@restackio/ai/event"; +import { ChatModel } from "openai/resources/index"; + +export async function openaiChatCompletionsStream({ + model = "gpt-4o-mini", + userName, + newMessage, + assistantName, + messages = [], + tools, + toolEvent, + streamAtCharacter, + streamEvent, + apiKey, + price, +}: { + model?: ChatModel; + userName?: string; + newMessage?: string; + assistantName?: string; + messages?: OpenAI.Chat.Completions.ChatCompletionMessageParam[]; + tools?: OpenAI.Chat.Completions.ChatCompletionTool[]; + toolEvent?: { + workflowEventName: string; + workflow?: SendWorkflowEvent["workflow"]; + }; + streamAtCharacter?: string; + streamEvent?: { + workflowEventName: string; + workflow?: SendWorkflowEvent["workflow"]; + }; + apiKey?: string; + price?: Price; +}) { + const restack = new Restack(); + const workflow = currentWorkflow().workflowExecution; + + log.debug("workflow", { workflow }); + + if (newMessage) { + messages.push({ + role: "user", + name: userName, + content: newMessage, + }); + } + + const openai = openaiClient({ apiKey }); + const chatStream = await openai.chat.completions.create({ + model: model, + messages, + tools, + stream: true, + stream_options: { + include_usage: true, + }, + }); + + const [stream, streamEnd] = chatStream.tee(); + const readableStream = streamEnd.toReadableStream() as unknown as ReadableStream; + const aggregatedStream = await aggregateStreamChunks(readableStream); + + let finishReason: ChatCompletionChunk.Choice["finish_reason"]; + let response: ChatCompletionChunk.Choice.Delta["content"] = ""; + let tokensCountInput = 0; + let tokensCountOutput = 0; + + for await (const chunk of stream) { + let content = chunk.choices[0]?.delta?.content || ""; + finishReason = chunk.choices[0]?.finish_reason; + tokensCountInput += chunk.usage?.prompt_tokens ?? 0; + tokensCountOutput += chunk.usage?.completion_tokens ?? 0; + + if (finishReason === "tool_calls") { + const { toolCalls } = mergeToolCalls(aggregatedStream); + await Promise.all( + toolCalls.map((toolCall) => { + if (toolEvent) { + const functionArguments = JSON.parse( + toolCall.function?.arguments ?? "" + ); + + const input: ToolCallEvent = { + ...toolCall, + function: { + name: toolCall.function?.name ?? "", + input: functionArguments, + }, + assistantName, + }; + + if (toolEvent) { + const workflowEvent = { + event: { + name: toolEvent.workflowEventName, + input, + }, + workflow: { + ...workflow, + ...toolEvent.workflow, + }, + }; + log.debug("toolEvent sendWorkflowEvent", { workflowEvent }); + + restack.sendWorkflowEvent(workflowEvent); + } + } + }) + ); + return { + result: { + messages, + toolCalls, + }, + cost: + price && + openaiCost({ + price, + tokensCount: { + input: tokensCountInput, + output: tokensCountOutput, + }, + }), + }; + } else { + response += content; + if ( + content.trim().slice(-1) === streamAtCharacter || + finishReason === "stop" + ) { + if (response.length) { + const input: StreamEvent = { + chunkId: chunk.id, + response, + assistantName, + isLast: finishReason === "stop", + }; + if (streamEvent) { + const workflowEvent = { + event: { + name: streamEvent.workflowEventName, + input, + }, + workflow: { + ...workflow, + ...streamEvent.workflow, + }, + }; + log.debug("streamEvent sendWorkflowEvent", { workflowEvent }); + restack.sendWorkflowEvent(workflowEvent); + } + } + } + + if (finishReason === "stop") { + const newMessage: OpenAI.Chat.Completions.ChatCompletionMessageParam = { + content: response, + role: "assistant", + name: assistantName, + }; + + messages.push(newMessage); + + return { + result: { + messages, + }, + cost: + price && + openaiCost({ + price, + tokensCount: { + input: tokensCountInput, + output: tokensCountOutput, + }, + }), + }; + } + } + } +} diff --git a/voice/src/functions/openai/chat/index.ts b/voice/src/functions/openai/chat/index.ts new file mode 100644 index 0000000..3cb4077 --- /dev/null +++ b/voice/src/functions/openai/chat/index.ts @@ -0,0 +1,2 @@ +export * from "./completionsBase"; +export * from "./completionsStream"; diff --git a/voice/src/functions/openai/index.ts b/voice/src/functions/openai/index.ts new file mode 100644 index 0000000..6a947d6 --- /dev/null +++ b/voice/src/functions/openai/index.ts @@ -0,0 +1,2 @@ +export * from "./chat"; +export * from "./thread"; diff --git a/voice/src/functions/openai/thread/createAssistant.ts b/voice/src/functions/openai/thread/createAssistant.ts new file mode 100644 index 0000000..f893399 --- /dev/null +++ b/voice/src/functions/openai/thread/createAssistant.ts @@ -0,0 +1,34 @@ +import { ChatModel } from "openai/resources/index"; +import { FunctionFailure } from "@restackio/ai/function"; +import { Assistant, AssistantTool } from "openai/resources/beta/index"; + +import { openaiClient } from "../utils/client"; + +export async function createAssistant({ + apiKey, + name, + instructions, + model = "gpt-4o-mini", + tools = [], +}: { + apiKey: string; + name: string; + instructions: string; + tools?: AssistantTool[]; + model: ChatModel; +}): Promise { + try { + const openai = openaiClient({ apiKey }); + + const assistant = await openai.beta.assistants.create({ + name, + instructions, + model, + tools, + }); + + return assistant; + } catch (error) { + throw FunctionFailure.nonRetryable(`Error creating assistant: ${error}`); + } +} diff --git a/voice/src/functions/openai/thread/createMessageOnThread.ts b/voice/src/functions/openai/thread/createMessageOnThread.ts new file mode 100644 index 0000000..63b4a24 --- /dev/null +++ b/voice/src/functions/openai/thread/createMessageOnThread.ts @@ -0,0 +1,28 @@ +import OpenAI from "openai/index"; +import { FunctionFailure } from "@restackio/ai/function"; + +import { openaiClient } from "../utils/client"; + +export async function createMessageOnThread({ + apiKey, + threadId, + content, + role, +}: { + apiKey: string; + threadId: string; + content: string; + role: OpenAI.Beta.Threads.MessageCreateParams["role"]; +}) { + try { + const openai = openaiClient({ apiKey }); + await openai.beta.threads.messages.create(threadId, { + role, + content, + }); + } catch (error) { + throw FunctionFailure.nonRetryable( + `Error creating message thread: ${error}` + ); + } +} diff --git a/voice/src/functions/openai/thread/createThread.ts b/voice/src/functions/openai/thread/createThread.ts new file mode 100644 index 0000000..5d29652 --- /dev/null +++ b/voice/src/functions/openai/thread/createThread.ts @@ -0,0 +1,19 @@ +import { FunctionFailure } from "@restackio/ai/function"; +import { Thread } from "openai/resources/beta/index"; + +import { openaiClient } from "../utils/client"; + +export async function createThread({ + apiKey, +}: { + apiKey: string; +}): Promise { + try { + const openai = openaiClient({ apiKey }); + const thread = await openai.beta.threads.create(); + + return thread; + } catch (error) { + throw FunctionFailure.nonRetryable(`Error creating thread: ${error}`); + } +} diff --git a/voice/src/functions/openai/thread/index.ts b/voice/src/functions/openai/thread/index.ts new file mode 100644 index 0000000..0ef24eb --- /dev/null +++ b/voice/src/functions/openai/thread/index.ts @@ -0,0 +1,4 @@ +export * from "./createAssistant"; +export * from "./createMessageOnThread"; +export * from "./createThread"; +export * from "./runThread"; diff --git a/voice/src/functions/openai/thread/runThread.ts b/voice/src/functions/openai/thread/runThread.ts new file mode 100644 index 0000000..57d17e6 --- /dev/null +++ b/voice/src/functions/openai/thread/runThread.ts @@ -0,0 +1,31 @@ +import { FunctionFailure } from "@restackio/ai/function"; +import { Stream } from "openai/streaming"; +import { AssistantStreamEvent } from "openai/resources/beta/index"; +import { Run } from "openai/resources/beta/threads/runs/index"; + +import { openaiClient } from "../utils/client"; + +export async function runThread({ + apiKey, + threadId, + assistantId, + stream = false, +}: { + apiKey: string; + threadId: string; + assistantId: string; + stream: boolean; +}): Promise | Run> { + try { + const openai = openaiClient({ apiKey }); + + const run = await openai.beta.threads.runs.create(threadId, { + assistant_id: assistantId, + ...(stream && { stream }), + }); + + return run; + } catch (error) { + throw FunctionFailure.nonRetryable(`Error running thread: ${error}`); + } +} \ No newline at end of file diff --git a/voice/src/functions/openai/types/events.ts b/voice/src/functions/openai/types/events.ts new file mode 100644 index 0000000..44b071b --- /dev/null +++ b/voice/src/functions/openai/types/events.ts @@ -0,0 +1,17 @@ +import OpenAI from "openai/index"; + +export type StreamEvent = { + chunkId?: string; + response: string; + assistantName?: string; + isLast: boolean; +}; + +export type ToolCallEvent = + OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall & { + function: { + name: string; + input: JSON; + }; + assistantName?: string; + }; diff --git a/voice/src/functions/openai/types/index.ts b/voice/src/functions/openai/types/index.ts new file mode 100644 index 0000000..1784004 --- /dev/null +++ b/voice/src/functions/openai/types/index.ts @@ -0,0 +1 @@ +export * from "./events"; diff --git a/voice/src/functions/openai/utils/aggregateStream.ts b/voice/src/functions/openai/utils/aggregateStream.ts new file mode 100644 index 0000000..18a7d74 --- /dev/null +++ b/voice/src/functions/openai/utils/aggregateStream.ts @@ -0,0 +1,26 @@ +export async function aggregateStreamChunks(stream: ReadableStream) { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (value) chunks.push(value); + } + + const aggregated = new Uint8Array( + chunks.reduce((acc, chunk) => acc + chunk.length, 0) + ); + let offset = 0; + for (const chunk of chunks) { + aggregated.set(chunk, offset); + offset += chunk.length; + } + + const textContent = new TextDecoder().decode(aggregated); + const jsonObjects = textContent + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line)); + return jsonObjects; +} diff --git a/voice/src/functions/openai/utils/client.ts b/voice/src/functions/openai/utils/client.ts new file mode 100644 index 0000000..77fd385 --- /dev/null +++ b/voice/src/functions/openai/utils/client.ts @@ -0,0 +1,21 @@ +import OpenAI from "openai/index"; +import "dotenv/config"; + +let openaiInstance: OpenAI | null = null; + +export const openaiClient = ({ + apiKey = process.env.OPENAI_API_KEY, +}: { + apiKey?: string; +}): OpenAI => { + if (!apiKey) { + throw new Error("API key is required to create OpenAI client."); + } + + if (!openaiInstance) { + openaiInstance = new OpenAI({ + apiKey, + }); + } + return openaiInstance; +}; diff --git a/voice/src/functions/openai/utils/cost.ts b/voice/src/functions/openai/utils/cost.ts new file mode 100644 index 0000000..2724ab6 --- /dev/null +++ b/voice/src/functions/openai/utils/cost.ts @@ -0,0 +1,24 @@ +export type TokensCount = { + input: number; + output: number; +}; + +export type Price = { + input: number; + output: number; +}; +export const openaiCost = ({ + tokensCount, + price, +}: { + tokensCount: TokensCount; + price: Price; +}): number => { + let cost = 0; + const { input: inputTokens, output: outputTokens } = tokensCount; + const { input: inputPrice, output: outputPrice } = price; + + cost = inputTokens * inputPrice + outputTokens * outputPrice; + + return cost; +}; diff --git a/voice/src/functions/openai/utils/index.ts b/voice/src/functions/openai/utils/index.ts new file mode 100644 index 0000000..0707ebb --- /dev/null +++ b/voice/src/functions/openai/utils/index.ts @@ -0,0 +1,4 @@ +export * from "./aggregateStream"; +export * from "./client"; +export * from "./cost"; +export * from "./mergeToolCalls"; diff --git a/voice/src/functions/openai/utils/mergeToolCalls.ts b/voice/src/functions/openai/utils/mergeToolCalls.ts new file mode 100644 index 0000000..600170f --- /dev/null +++ b/voice/src/functions/openai/utils/mergeToolCalls.ts @@ -0,0 +1,31 @@ +import OpenAI from "openai/index"; +import { ChatCompletionChunk } from "openai/resources/chat/completions.mjs"; + +export function mergeToolCalls(aggregatedStream: ChatCompletionChunk[]) { + const toolCalls: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall[] = + []; + + aggregatedStream.forEach((chunk) => { + chunk.choices.forEach((choice) => { + if (choice.delta.tool_calls) { + choice.delta.tool_calls.forEach((toolCall) => { + const lastToolCall = toolCalls[toolCalls.length - 1]; + if (toolCall.id) { + toolCalls.push({ + ...toolCall, + function: { ...toolCall.function, arguments: "" }, + }); + } else if ( + lastToolCall && + lastToolCall.function && + toolCall.function?.arguments + ) { + lastToolCall.function.arguments += toolCall.function.arguments; + } + }); + } + }); + }); + + return { toolCalls }; +} diff --git a/voice/src/functions/twilio/call.ts b/voice/src/functions/twilio/call.ts new file mode 100644 index 0000000..97efd3e --- /dev/null +++ b/voice/src/functions/twilio/call.ts @@ -0,0 +1,34 @@ +import { FunctionFailure } from "@restackio/ai/function"; +import { CallListInstanceCreateOptions } from "twilio/lib/rest/api/v2010/account/call"; +import { twilioClient } from "./utils/client"; + +interface Output { + sid: string; +} + +export async function twilioCall({ + accountSid, + authToken, + options, +}: { + accountSid?: string; + authToken?: string; + options: CallListInstanceCreateOptions; +}): Promise { + const client = twilioClient({ accountSid, authToken }); + + if (!accountSid || !authToken) { + throw FunctionFailure.nonRetryable("Twilio credentials are missing"); + } + + try { + if (options.to && options.from && options.url) { + const { sid } = await client.calls.create(options); + return { sid }; + } else { + throw FunctionFailure.nonRetryable(`No to, from or url`); + } + } catch (error) { + throw FunctionFailure.nonRetryable(`Error Twilio call create: ${error}`); + } +} diff --git a/voice/src/functions/twilio/index.ts b/voice/src/functions/twilio/index.ts new file mode 100644 index 0000000..e575c1a --- /dev/null +++ b/voice/src/functions/twilio/index.ts @@ -0,0 +1 @@ +export * from "./call"; diff --git a/voice/src/functions/twilio/utils/client.ts b/voice/src/functions/twilio/utils/client.ts new file mode 100644 index 0000000..befe8e6 --- /dev/null +++ b/voice/src/functions/twilio/utils/client.ts @@ -0,0 +1,23 @@ +import twilio from "twilio/index"; +import "dotenv/config"; + +let clientTwilio: twilio.Twilio; + +export function twilioClient({ + accountSid = process.env.TWILIO_ACCOUNT_SID, + authToken = process.env.TWILIO_AUTH_TOKEN, +}: { + accountSid: string; + authToken: string; +}) { + if (!accountSid || !authToken) { + throw new Error( + "Account SID and auth token are required to create Twilio client." + ); + } + + if (!clientTwilio) { + clientTwilio = twilio(accountSid, authToken); + } + return clientTwilio; +} diff --git a/voice/src/functions/websocket/index.ts b/voice/src/functions/websocket/index.ts new file mode 100644 index 0000000..b561575 --- /dev/null +++ b/voice/src/functions/websocket/index.ts @@ -0,0 +1,2 @@ +export * from "./listen"; +export * from "./send"; diff --git a/voice/src/functions/websocket/listen.ts b/voice/src/functions/websocket/listen.ts new file mode 100644 index 0000000..4c4034f --- /dev/null +++ b/voice/src/functions/websocket/listen.ts @@ -0,0 +1,64 @@ +import { heartbeat, currentWorkflow, log } from "@restackio/ai/function"; +import { websocketConnect } from "./utils/client"; +import Restack from "@restackio/ai"; +import { SendWorkflowEvent } from "@restackio/ai/event"; + +export async function websocketListen({ + streamSid, + events, + address, +}: { + streamSid: string; + events?: { + websocketEventName: string; + workflowEventName: string; + workflow?: SendWorkflowEvent["workflow"]; + }[]; + address?: string; +}) { + return new Promise(async (resolve) => { + const ws = await websocketConnect({ address }); + + const restack = new Restack(); + const workflow = currentWorkflow().workflowExecution; + + ws.on("message", (data) => { + const message = JSON.parse(data.toString()); + if (message.streamSid === streamSid) { + if (events) { + events.forEach((listenEvent) => { + if (message.event === listenEvent.websocketEventName) { + if (message.media && message.media.track !== "inbound") { + return; + } + + const workflowEvent: SendWorkflowEvent = { + event: { + name: listenEvent.workflowEventName, + input: { + streamSid, + data: message.data, + media: message.media, + }, + }, + workflow: { + ...workflow, + ...listenEvent.workflow, + }, + }; + log.debug(`${message.event} sendWorkflowEvent`, { + workflowEvent, + }); + + restack.sendWorkflowEvent(workflowEvent); + } + }); + } + heartbeat(message.streamSid); + if (message.event === "stop") { + resolve(); + } + } + }); + }); +} diff --git a/voice/src/functions/websocket/send.ts b/voice/src/functions/websocket/send.ts new file mode 100644 index 0000000..00f4896 --- /dev/null +++ b/voice/src/functions/websocket/send.ts @@ -0,0 +1,27 @@ +import { WebsocketEvent } from "websocket/types"; +import { websocketConnect } from "./utils/client"; + +export async function websocketSend({ + name, + input, + address, +}: { + name: string; + input: WebsocketEvent; + address?: string; +}) { + const ws = await websocketConnect({ address }); + + const { streamSid, data, media } = input; + + const event = { + streamSid, + event: name, + data, + media, + }; + + ws.send(JSON.stringify(event)); + ws.close(); + return true; +} diff --git a/voice/src/functions/websocket/types/events.ts b/voice/src/functions/websocket/types/events.ts new file mode 100644 index 0000000..4759b96 --- /dev/null +++ b/voice/src/functions/websocket/types/events.ts @@ -0,0 +1,12 @@ +export type WebsocketEvent = { + streamSid: string; // For Twilio compatibility + media?: { + track?: "inbound" | "outbound"; // For Twilio compatibility + trackId: string; + payload?: string; + }; + data?: { + trackId: string; + [key: string]: any; + }; +}; diff --git a/voice/src/functions/websocket/types/index.ts b/voice/src/functions/websocket/types/index.ts new file mode 100644 index 0000000..1784004 --- /dev/null +++ b/voice/src/functions/websocket/types/index.ts @@ -0,0 +1 @@ +export * from "./events"; diff --git a/voice/src/functions/websocket/utils/client.ts b/voice/src/functions/websocket/utils/client.ts new file mode 100644 index 0000000..3611c02 --- /dev/null +++ b/voice/src/functions/websocket/utils/client.ts @@ -0,0 +1,31 @@ +import WebSocket from "ws"; +import "dotenv/config"; +import { FunctionFailure } from "@restackio/ai/function"; + +export function websocketConnect({ + address = process.env.WEBSOCKET_ADDRESS, +}: { + address?: string; +}): Promise { + return new Promise((resolve, reject) => { + try { + const ws = new WebSocket(address); + + ws.on("open", () => { + resolve(ws); + }); + + ws.on("error", (error) => { + reject( + FunctionFailure.nonRetryable( + `Error connecting to WebSocket: ${error}` + ) + ); + }); + } catch (error) { + reject( + FunctionFailure.nonRetryable(`Error connecting to WebSocket: ${error}`) + ); + } + }); +} diff --git a/voice/src/services.ts b/voice/src/services.ts index ccfea7e..57b4c49 100644 --- a/voice/src/services.ts +++ b/voice/src/services.ts @@ -4,11 +4,14 @@ import { erpPlaceOrder, erpCheckInventory, erpCheckPrice, + websocketSend, + websocketListen, + twilioCall, + openaiChatCompletionsBase, + openaiChatCompletionsStream, + deepgramListen, + deepgramSpeak, } from "./functions"; -import { websocketService } from "@restackio/integrations-websocket"; -import { twilioService } from "@restackio/integrations-twilio"; -import { openaiService } from "@restackio/integrations-openai"; -import { deepgramService } from "@restackio/integrations-deepgram"; import { client } from "./client"; export async function services() { @@ -29,10 +32,34 @@ export async function services() { erpPlaceOrder, }, }), - websocketService({ client }), - twilioService({ client }), - openaiService({ client }), - deepgramService({ client }), + client.startService({ + taskQueue: "websocket", + functions: { + websocketSend, + websocketListen, + }, + }), + client.startService({ + taskQueue: "twilio", + functions: { + twilioCall, + }, + }), + client.startService({ + taskQueue: "openai", + functions: { + openaiChatCompletionsBase, + openaiChatCompletionsStream, + }, + }), + client.startService({ + taskQueue: "deepgram", + functions: { + deepgramListen, + deepgramSpeak, + }, + }), + ]); console.log("Services running successfully."); diff --git a/voice/src/workflows/conversation/conversation.ts b/voice/src/workflows/conversation/conversation.ts index 6ab24e4..a73b59f 100644 --- a/voice/src/workflows/conversation/conversation.ts +++ b/voice/src/workflows/conversation/conversation.ts @@ -2,14 +2,12 @@ import { step, log, workflowInfo, condition } from "@restackio/ai/workflow"; import * as functions from "../../functions"; import { onEvent } from "@restackio/ai/event"; import { streamEvent, toolCallEvent, conversationEndEvent } from "./events"; -import { openaiTaskQueue } from "@restackio/integrations-openai/taskQueue"; -import * as openaiFunctions from "@restackio/integrations-openai/functions"; import { UserEvent, userEvent } from "../room/events"; import { StreamEvent, ToolCallEvent, -} from "@restackio/integrations-openai/types"; +} from "../../functions/openai/types"; import { agentPrompt } from "../../functions/openai/prompt"; import { ChatModel, @@ -53,8 +51,8 @@ export async function conversationWorkflow({ }, }; - const { result } = await step({ - taskQueue: openaiTaskQueue, + const { result } = await step({ + taskQueue: "openai", }).openaiChatCompletionsStream({ userName, newMessage: message, @@ -69,8 +67,8 @@ export async function conversationWorkflow({ // On user event, send it to AI chat with previous messages to continue conversation. onEvent(userEvent, async ({ message, userName }: UserEvent) => { - const { result } = await step({ - taskQueue: openaiTaskQueue, + const { result } = await step({ + taskQueue: "openai", }).openaiChatCompletionsStream({ newMessage: message, userName, @@ -152,8 +150,8 @@ export async function conversationWorkflow({ name: toolFunction.name, }); - const { result } = await step({ - taskQueue: openaiTaskQueue, + const { result } = await step({ + taskQueue: "openai", }).openaiChatCompletionsStream({ messages: openaiChatMessages, ...commonOpenaiOptions, diff --git a/voice/src/workflows/room/events.ts b/voice/src/workflows/room/events.ts index 36a6bb8..7775584 100644 --- a/voice/src/workflows/room/events.ts +++ b/voice/src/workflows/room/events.ts @@ -1,5 +1,5 @@ import { defineEvent } from "@restackio/ai/event"; -import { WebsocketEvent } from "@restackio/integrations-websocket/types"; +import { WebsocketEvent } from "../functions/websocket/types"; export type RoomInfo = { streamSid: string; diff --git a/voice/src/workflows/room/room.ts b/voice/src/workflows/room/room.ts index 0e4a634..56817f8 100644 --- a/voice/src/workflows/room/room.ts +++ b/voice/src/workflows/room/room.ts @@ -18,12 +18,8 @@ import { roomMessageEvent, } from "./events"; import { streamEvent } from "../conversation/events"; -import { websocketTaskQueue } from "@restackio/integrations-websocket/taskQueue"; -import * as websocketFunctions from "@restackio/integrations-websocket/functions"; -import { deepgramTaskQueue } from "@restackio/integrations-deepgram/taskQueue"; -import * as deepgramFunctions from "@restackio/integrations-deepgram/functions"; -import { StreamEvent } from "@restackio/integrations-openai/types"; -import { WebsocketEvent } from "@restackio/integrations-websocket/types"; +import { StreamEvent } from "../../functions/openai/types"; +import { WebsocketEvent } from "../../functions/websocket/types"; export async function roomWorkflow({ address }: { address?: string }) { try { @@ -41,8 +37,8 @@ export async function roomWorkflow({ address }: { address?: string }) { // Start long running websocket and stream welcome message to websocket. onEvent(streamInfoEvent, async ({ streamSid }: RoomInfo) => { log.info(`Workflow update with streamSid: ${streamSid}`); - step({ - taskQueue: websocketTaskQueue, + step({ + taskQueue: "websocket", scheduleToCloseTimeout: "1 hour", heartbeatTimeout: "2 minutes", }).websocketListen({ @@ -62,15 +58,15 @@ export async function roomWorkflow({ address }: { address?: string }) { const welcomeMessage = "Hello! I am Pete from Apple."; - const { media } = await step({ - taskQueue: deepgramTaskQueue, + const { media } = await step({ + taskQueue: "deepgram", }).deepgramSpeak({ text: welcomeMessage, twilioEncoding: true, }); - await step({ - taskQueue: websocketTaskQueue, + await step({ + taskQueue: "websocket", }).websocketSend({ name: "media", input: { @@ -83,8 +79,8 @@ export async function roomWorkflow({ address }: { address?: string }) { address, }); - await step({ - taskQueue: websocketTaskQueue, + await step({ + taskQueue: "websocket", }).websocketSend({ name: roomMessageEvent.name, input: { @@ -105,8 +101,8 @@ export async function roomWorkflow({ address }: { address?: string }) { if (!media?.payload || media.trackId === assistantName) return; - const { result } = await step({ - taskQueue: deepgramTaskQueue, + const { result } = await step({ + taskQueue: "deepgram", }).deepgramListen({ base64Payload: media?.payload, twilioEncoding: true, @@ -137,8 +133,8 @@ export async function roomWorkflow({ address }: { address?: string }) { interactionCount += 1; - step({ - taskQueue: websocketTaskQueue, + step({ + taskQueue: "websocket", }).websocketSend({ name: userEvent.name, input: { @@ -189,8 +185,8 @@ export async function roomWorkflow({ address }: { address?: string }) { onEvent( streamEvent, async ({ response, isLast, assistantName }: StreamEvent) => { - const { media } = await step({ - taskQueue: deepgramTaskQueue, + const { media } = await step({ + taskQueue: "deepgram", }).deepgramSpeak({ text: response, twilioEncoding: true, @@ -204,8 +200,8 @@ export async function roomWorkflow({ address }: { address?: string }) { while (audioQueue.length > 0) { const { audio } = audioQueue.shift()!; - await step({ - taskQueue: websocketTaskQueue, + await step({ + taskQueue: "websocket", }).websocketSend({ name: "media", input: { @@ -219,8 +215,8 @@ export async function roomWorkflow({ address }: { address?: string }) { }); } - await step({ - taskQueue: websocketTaskQueue, + await step({ + taskQueue: "websocket", }).websocketSend({ name: roomMessageEvent.name, input: { diff --git a/voice/src/workflows/twilioCall.ts b/voice/src/workflows/twilioCall.ts index 3d9bd92..fcf754e 100644 --- a/voice/src/workflows/twilioCall.ts +++ b/voice/src/workflows/twilioCall.ts @@ -1,6 +1,5 @@ import { log, step } from "@restackio/ai/workflow"; -import * as twilioFunctions from "@restackio/integrations-twilio/functions"; -import { twilioTaskQueue } from "@restackio/integrations-twilio/taskQueue"; +import * as functions from "../functions"; interface Output { sid: string; } @@ -14,8 +13,8 @@ export async function twilioCallWorkflow({ from: string; url: string; }): Promise { - const { sid } = await step({ - taskQueue: twilioTaskQueue, + const { sid } = await step({ + taskQueue: "twilio", scheduleToCloseTimeout: "1 minute", }).twilioCall({ options: { From 86ffe216f88df932339ad417e6ab71e4522014b9 Mon Sep 17 00:00:00 2001 From: martinibach Date: Mon, 25 Nov 2024 12:41:22 +0100 Subject: [PATCH 2/7] fix build --- voice/src/functions/deepgram/listen.ts | 8 ++--- voice/src/functions/deepgram/speak.ts | 2 +- voice/src/functions/deepgram/utils/client.ts | 7 ++-- voice/src/functions/twilio/utils/client.ts | 4 +-- voice/src/functions/websocket/send.ts | 2 +- voice/src/functions/websocket/utils/client.ts | 2 +- .../workflows/conversation/conversation.ts | 36 ++++++++----------- voice/src/workflows/conversation/events.ts | 2 +- voice/src/workflows/room/events.ts | 2 +- voice/src/workflows/room/room.ts | 12 +++---- 10 files changed, 33 insertions(+), 44 deletions(-) diff --git a/voice/src/functions/deepgram/listen.ts b/voice/src/functions/deepgram/listen.ts index 341e5bc..fc1d0ef 100644 --- a/voice/src/functions/deepgram/listen.ts +++ b/voice/src/functions/deepgram/listen.ts @@ -13,12 +13,10 @@ export async function deepgramListen({ utterance_end_ms: 2000, }, twilioEncoding, - apiKey, }: { base64Payload: string; options?: PrerecordedSchema; twilioEncoding?: boolean; - apiKey?: string; }) { if (!base64Payload) { throw FunctionFailure.nonRetryable("No audio file"); @@ -27,7 +25,7 @@ export async function deepgramListen({ try { const decodedBuffer = Buffer.from(base64Payload, "base64"); - const deepgram = deepgramClient({ apiKey }); + const deepgram = deepgramClient(); const response = await deepgram.listen.prerecorded.transcribeFile( decodedBuffer, @@ -45,13 +43,13 @@ export async function deepgramListen({ } const result = response.result; - const firstChannel = result.results?.channels?.[0]; + const firstChannel = result?.results?.channels?.[0]; const transcript = firstChannel?.alternatives?.[0]?.transcript; let language = ""; if (options.detect_language) { - language = firstChannel?.detected_language; + language = firstChannel?.detected_language!; } return { diff --git a/voice/src/functions/deepgram/speak.ts b/voice/src/functions/deepgram/speak.ts index 1037bf4..7dad060 100644 --- a/voice/src/functions/deepgram/speak.ts +++ b/voice/src/functions/deepgram/speak.ts @@ -42,7 +42,7 @@ export async function deepgramSpeak({ } try { - const deepgram = deepgramClient({ apiKey }); + const deepgram = deepgramClient(); const response = await deepgram.speak.request( { text }, { diff --git a/voice/src/functions/deepgram/utils/client.ts b/voice/src/functions/deepgram/utils/client.ts index 8564c7a..755c9c1 100644 --- a/voice/src/functions/deepgram/utils/client.ts +++ b/voice/src/functions/deepgram/utils/client.ts @@ -3,11 +3,8 @@ import "dotenv/config"; let clientDeepgram: DeepgramClient; -export function deepgramClient({ - apiKey = process.env.DEEPGRAM_API_KEY, -}: { - apiKey: string; -}) { +export function deepgramClient() { + const apiKey = process.env.DEEPGRAM_API_KEY; if (!apiKey) { throw new Error("API key is required to create Deepgram client."); } diff --git a/voice/src/functions/twilio/utils/client.ts b/voice/src/functions/twilio/utils/client.ts index befe8e6..8c1b182 100644 --- a/voice/src/functions/twilio/utils/client.ts +++ b/voice/src/functions/twilio/utils/client.ts @@ -7,8 +7,8 @@ export function twilioClient({ accountSid = process.env.TWILIO_ACCOUNT_SID, authToken = process.env.TWILIO_AUTH_TOKEN, }: { - accountSid: string; - authToken: string; + accountSid?: string; + authToken?: string; }) { if (!accountSid || !authToken) { throw new Error( diff --git a/voice/src/functions/websocket/send.ts b/voice/src/functions/websocket/send.ts index 00f4896..3fdff5b 100644 --- a/voice/src/functions/websocket/send.ts +++ b/voice/src/functions/websocket/send.ts @@ -1,4 +1,4 @@ -import { WebsocketEvent } from "websocket/types"; +import { WebsocketEvent } from "./types"; import { websocketConnect } from "./utils/client"; export async function websocketSend({ diff --git a/voice/src/functions/websocket/utils/client.ts b/voice/src/functions/websocket/utils/client.ts index 3611c02..34a4b17 100644 --- a/voice/src/functions/websocket/utils/client.ts +++ b/voice/src/functions/websocket/utils/client.ts @@ -9,7 +9,7 @@ export function websocketConnect({ }): Promise { return new Promise((resolve, reject) => { try { - const ws = new WebSocket(address); + const ws = new WebSocket(address!); ws.on("open", () => { resolve(ws); diff --git a/voice/src/workflows/conversation/conversation.ts b/voice/src/workflows/conversation/conversation.ts index a73b59f..56303f8 100644 --- a/voice/src/workflows/conversation/conversation.ts +++ b/voice/src/workflows/conversation/conversation.ts @@ -12,6 +12,8 @@ import { agentPrompt } from "../../functions/openai/prompt"; import { ChatModel, ChatCompletionAssistantMessageParam, + ChatCompletionMessageParam, + ChatCompletionTool, } from "openai/resources/index"; export async function conversationWorkflow({ @@ -27,9 +29,7 @@ export async function conversationWorkflow({ const parentWorkflow = workflowInfo().parent; if (!parentWorkflow) throw "no parent Workflow"; - let openaiChatMessages: any[] = []; - - // Get tools definition and start conversation. + let openaiChatMessages: ChatCompletionMessageParam[] = []; const tools = await step({ taskQueue: "erp", @@ -51,7 +51,7 @@ export async function conversationWorkflow({ }, }; - const { result } = await step({ + const response = await step({ taskQueue: "openai", }).openaiChatCompletionsStream({ userName, @@ -60,14 +60,12 @@ export async function conversationWorkflow({ ...commonOpenaiOptions, }); - if (result.messages) { - openaiChatMessages = result.messages; + if (response?.result?.messages) { + openaiChatMessages = response.result.messages; } - // On user event, send it to AI chat with previous messages to continue conversation. - onEvent(userEvent, async ({ message, userName }: UserEvent) => { - const { result } = await step({ + const response = await step({ taskQueue: "openai", }).openaiChatCompletionsStream({ newMessage: message, @@ -76,12 +74,12 @@ export async function conversationWorkflow({ ...commonOpenaiOptions, }); - if (result.messages) { - openaiChatMessages = result.messages; + if (response?.result?.messages) { + openaiChatMessages = response.result.messages; } - if (result.toolCalls) { - result.toolCalls.map(async (toolCall) => { + if (response?.result?.toolCalls) { + response.result.toolCalls.map(async (toolCall) => { const toolResponse = `Sure, let me ${toolCall?.function?.name}...`; const toolMessage: ChatCompletionAssistantMessageParam = { content: toolResponse, @@ -110,8 +108,6 @@ export async function conversationWorkflow({ return { message }; }); - // When AI answer is a tool call, execute function and push results to conversation. - onEvent( toolCallEvent, async ({ function: toolFunction }: ToolCallEvent) => { @@ -150,23 +146,21 @@ export async function conversationWorkflow({ name: toolFunction.name, }); - const { result } = await step({ + const response = await step({ taskQueue: "openai", }).openaiChatCompletionsStream({ messages: openaiChatMessages, ...commonOpenaiOptions, }); - if (result.messages) { - openaiChatMessages = result.messages; + if (response?.result?.messages) { + openaiChatMessages = response.result.messages; } return { function: toolFunction }; } ); - // Terminate conversation workflow. - let ended = false; onEvent(conversationEndEvent, async () => { log.info(`conversationEndEvent received`); @@ -180,4 +174,4 @@ export async function conversationWorkflow({ log.error("Error in conversationWorkflow", { error }); throw error; } -} +} \ No newline at end of file diff --git a/voice/src/workflows/conversation/events.ts b/voice/src/workflows/conversation/events.ts index f7d61c4..6ceb2f0 100644 --- a/voice/src/workflows/conversation/events.ts +++ b/voice/src/workflows/conversation/events.ts @@ -2,7 +2,7 @@ import { defineEvent } from "@restackio/ai/event"; import { StreamEvent, ToolCallEvent, -} from "@restackio/integrations-openai/types"; +} from "../../functions/openai/types"; export const streamEvent = defineEvent("stream"); export const toolCallEvent = defineEvent("toolCall"); diff --git a/voice/src/workflows/room/events.ts b/voice/src/workflows/room/events.ts index 7775584..31da75e 100644 --- a/voice/src/workflows/room/events.ts +++ b/voice/src/workflows/room/events.ts @@ -1,5 +1,5 @@ import { defineEvent } from "@restackio/ai/event"; -import { WebsocketEvent } from "../functions/websocket/types"; +import { WebsocketEvent } from "../../functions/websocket/types"; export type RoomInfo = { streamSid: string; diff --git a/voice/src/workflows/room/room.ts b/voice/src/workflows/room/room.ts index 56817f8..e718139 100644 --- a/voice/src/workflows/room/room.ts +++ b/voice/src/workflows/room/room.ts @@ -108,9 +108,9 @@ export async function roomWorkflow({ address }: { address?: string }) { twilioEncoding: true, }); - const transcript = result.results.channels[0].alternatives[0].transcript; + const transcript = result?.results.channels[0].alternatives[0].transcript; - if (!transcript.length) { + if (!transcript?.length) { const input: StreamEvent = { response: "Sorry i didn't understand. Can you repeat?", assistantName, @@ -153,7 +153,7 @@ export async function roomWorkflow({ address }: { address?: string }) { { assistantName, userName: media.trackId, - message: transcript, + message: transcript!, }, ], workflowId: `${streamSid}-conversationWorkflow`, @@ -162,7 +162,7 @@ export async function roomWorkflow({ address }: { address?: string }) { } else { const input: UserEvent = { userName: media.trackId, - message: transcript, + message: transcript!, }; step({ taskQueue: `restack`, @@ -207,7 +207,7 @@ export async function roomWorkflow({ address }: { address?: string }) { input: { streamSid: currentstreamSid, media: { - trackId: assistantName, + trackId: assistantName!, payload: audio, }, }, @@ -221,7 +221,7 @@ export async function roomWorkflow({ address }: { address?: string }) { name: roomMessageEvent.name, input: { streamSid: currentstreamSid, - data: { trackId: assistantName, text: response }, + data: { trackId: assistantName!, text: response }, }, address, }); From c5d68a6c1fb27b0820a641f984622da23fc84677 Mon Sep 17 00:00:00 2001 From: martinibach Date: Wed, 27 Nov 2024 18:18:43 +0100 Subject: [PATCH 3/7] Refactor import path in callWorkflow.ts and add console.log in room.ts --- voice/callWorkflow.ts | 2 +- voice/package.json | 17 +- voice/pnpm-lock.yaml | 389 ++++++++++++++++++++++++++++--- voice/restack_up.mjs | 38 ++- voice/src/workflows/room/room.ts | 1 + 5 files changed, 393 insertions(+), 54 deletions(-) diff --git a/voice/callWorkflow.ts b/voice/callWorkflow.ts index f046769..3d55afa 100644 --- a/voice/callWorkflow.ts +++ b/voice/callWorkflow.ts @@ -1,4 +1,4 @@ -import { client } from "../hello/src/client"; +import { client } from "./src/client"; async function scheduleWorkflow() { try { diff --git a/voice/package.json b/voice/package.json index 01a6a8b..f99be11 100644 --- a/voice/package.json +++ b/voice/package.json @@ -11,14 +11,19 @@ "build": "tsc --build", "clean": "rm -rf node_modules", "call": "ts-node ./callWorkflow.ts", - "ngrok": "ngrok http 4000" + "ngrok": "ngrok http 4000", + "restack-up": "dotenv -e .env -- tsx restack_up.mjs", + "docker:build:server": "docker build -f Dockerfile.server -t voice .", + "docker:run:server": "docker run -d -p 4000:4000 voice", + "docker:build:services": "docker build -f Dockerfile.services -t voice .", + "docker:run:services": "docker run -d -p 4000:4000 voice" }, "keywords": [], "author": "", "license": "ISC", "dependencies": { - "@restackio/ai": "0.0.85", - "@temporalio/workflow": "1.11.1", + "@restackio/ai": "0.0.86", + "@temporalio/workflow": "1.11.5", "cors": "2.8.5", "dotenv": "16.4.5", "express": "4.21.0", @@ -32,14 +37,16 @@ "zod": "3.23.8" }, "devDependencies": { - "@restackio/cloud": "1.0.19", + "@restackio/cloud": "1.0.21", "@types/cors": "2.8.17", "@types/express": "4.17.21", "@types/node": "22.5.4", "@types/uuid": "10.0.0", "@types/ws": "8.5.12", "nodemon": "3.1.4", - "ts-node-dev": "2.0.0" + "ts-node-dev": "2.0.0", + "tsx": "4.19.2", + "dotenv-cli": "7.4.4" }, "optionalDependencies": { "bufferutil": "4.0.8" diff --git a/voice/pnpm-lock.yaml b/voice/pnpm-lock.yaml index 59de3ad..3863d3f 100644 --- a/voice/pnpm-lock.yaml +++ b/voice/pnpm-lock.yaml @@ -12,11 +12,11 @@ importers: specifier: 3.6.0 version: 3.6.0(bufferutil@4.0.8) '@restackio/ai': - specifier: 0.0.85 - version: 0.0.85 + specifier: 0.0.86 + version: 0.0.86 '@temporalio/workflow': - specifier: 1.11.1 - version: 1.11.1 + specifier: 1.11.5 + version: 1.11.5 cors: specifier: 2.8.5 version: 2.8.5 @@ -53,8 +53,8 @@ importers: version: 4.0.8 devDependencies: '@restackio/cloud': - specifier: 1.0.19 - version: 1.0.19 + specifier: 1.0.21 + version: 1.0.21 '@types/cors': specifier: 2.8.17 version: 2.8.17 @@ -70,12 +70,18 @@ importers: '@types/ws': specifier: 8.5.12 version: 8.5.12 + dotenv-cli: + specifier: 7.4.4 + version: 7.4.4 nodemon: specifier: 3.1.4 version: 3.1.4 ts-node-dev: specifier: 2.0.0 version: 2.0.0(@swc/core@1.9.3)(@types/node@22.5.4)(typescript@5.6.2) + tsx: + specifier: 4.19.2 + version: 4.19.2 packages: @@ -91,6 +97,150 @@ packages: resolution: {integrity: sha512-uErkoOyBjdk3u4UyowxN6LtjuLv8s1V/29WVeR2Y2M1wKGo93s8dlYFsF3R1SWeMaeJOimpCcwjWvMQbYrmS6Q==} engines: {node: '>=18.0.0'} + '@esbuild/aix-ppc64@0.23.1': + resolution: {integrity: sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.23.1': + resolution: {integrity: sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.23.1': + resolution: {integrity: sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.23.1': + resolution: {integrity: sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.23.1': + resolution: {integrity: sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.23.1': + resolution: {integrity: sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.23.1': + resolution: {integrity: sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.23.1': + resolution: {integrity: sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.23.1': + resolution: {integrity: sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.23.1': + resolution: {integrity: sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.23.1': + resolution: {integrity: sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.23.1': + resolution: {integrity: sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.23.1': + resolution: {integrity: sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.23.1': + resolution: {integrity: sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.23.1': + resolution: {integrity: sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.23.1': + resolution: {integrity: sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.23.1': + resolution: {integrity: sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-x64@0.23.1': + resolution: {integrity: sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.23.1': + resolution: {integrity: sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.23.1': + resolution: {integrity: sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/sunos-x64@0.23.1': + resolution: {integrity: sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.23.1': + resolution: {integrity: sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.23.1': + resolution: {integrity: sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.23.1': + resolution: {integrity: sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@grpc/grpc-js@1.12.2': resolution: {integrity: sha512-bgxdZmgTrJZX50OjyVwz3+mNEnCTNkh3cIqGPWVNeW9jX6bn1ZkU80uPd+67/ZpIJIjRQ9qaHCjhavyoWYxumg==} engines: {node: '>=12.10.0'} @@ -181,12 +331,12 @@ packages: '@protobufjs/utf8@1.1.0': resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} - '@restackio/ai@0.0.85': - resolution: {integrity: sha512-UWDcywhNjcTn7TbgqJL1ahzpf722RZV57rnR/b5s48eMgUHZNNxMJWDvCYIW5IZKEvzLp124iu0FNrV9UbKTpA==} + '@restackio/ai@0.0.86': + resolution: {integrity: sha512-i4krGAb6mGRhk+5+H81uUK+mfu8DDBGtqb+HaSDq2uXDflrmbO+CrNbMfv5WMQKH6GMjM/RNw5CO7f6be48CsA==} engines: {node: '>=20'} - '@restackio/cloud@1.0.19': - resolution: {integrity: sha512-UJNfM4di6qGa3EqAZzUdcrGjK6tIj2YmQbGZouBZjpHDqG3kbzMZkgNPcW/UsCp0HYCKS36pUmDy6m1GkMFgNQ==} + '@restackio/cloud@1.0.21': + resolution: {integrity: sha512-KZTl9vww0eGoAVf7TKmOIk9fJ+OIlLzwjLzJb2tILFsye08nvmEvVM5KW8oCxyeXxtAyATYsysqs626XND5p1g==} engines: {node: '>=18'} '@swc/core-darwin-arm64@1.9.3': @@ -270,18 +420,12 @@ packages: '@temporalio/client@1.11.5': resolution: {integrity: sha512-Q8hFqxyf41MQmCeyxcGpbiMyuxhilvG8FScFwSbFs8ZZPZykNcpr5o+TOowLWzGWAL5f2bIKETh/eXDZpZvY7g==} - '@temporalio/common@1.11.1': - resolution: {integrity: sha512-8fQwmYHMakSbqMX26FNfiLwBwVwsMNbGmVfGaFOqy72QZuUiXxgmImXQhK6ir1ieeH8bLyY5laAosB4W6VPBAA==} - '@temporalio/common@1.11.5': resolution: {integrity: sha512-6cgGTAT+jSKKwCPOoUDIseJuDroP7cEIAX/pYpNBRGvfj+lpU8GitSVsPEZTVoMQ400otzBa1n80aH8bnOLVTw==} '@temporalio/core-bridge@1.11.5': resolution: {integrity: sha512-1IzxtPrndR99golP1yZnvsOHU9QP5o9KdNtM/W5VxChFc0ihmeZpisptgfYyqa9VaO/Bs9wEoSmH6fCSB9knTQ==} - '@temporalio/proto@1.11.1': - resolution: {integrity: sha512-qTNSsU88IOJbv6lhWyW0qAHAYg4euEW/NNPgAp5ohfPkqaw+/j51QABO9E6m/KQwMZ6x43z2JjQRS37aC6Psiw==} - '@temporalio/proto@1.11.5': resolution: {integrity: sha512-LjRGQdLRpRxDp2NSyNyhCp7JLaUlMY2T+hAeGfueR5cOVZxHXTO8TXnnjimi0UM1knyA6sW3yNJCNieKlCcASg==} @@ -289,9 +433,6 @@ packages: resolution: {integrity: sha512-WeIK/2ZQMCFVz89FeSa081kMDeHbyL0yZf335Rn0my2l0TcWWoHjVI9kQGuEAuP6fke/sCCQxpcxJ8EvEMHX8w==} engines: {node: '>= 16.0.0'} - '@temporalio/workflow@1.11.1': - resolution: {integrity: sha512-aTfoHPc6O/7UioOSe+PHmpzLZveG2qn0VI/snlclPFs8VO8r5hcbXb5jYeuzCi26mm7Z+Dxmf2hCK77tJlPeKg==} - '@temporalio/workflow@1.11.5': resolution: {integrity: sha512-U0kGoQ6bttEiT59km+cpeSxVr2Bq8ZHmlurP2N+9nQdxa10RsMqrp5YQrJQfg7KrfrYbeY1HyJBQ7Ljy1FrOtQ==} @@ -594,6 +735,10 @@ packages: cross-fetch@3.1.8: resolution: {integrity: sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==} + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + dayjs@1.11.13: resolution: {integrity: sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==} @@ -650,6 +795,14 @@ packages: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} + dotenv-cli@7.4.4: + resolution: {integrity: sha512-XkBYCG0tPIes+YZr4SpfFv76SQrV/LeCE8CI7JSEMi3VR9MvTihCGTOtbIexD6i2mXF+6px7trb1imVCXSNMDw==} + hasBin: true + + dotenv-expand@10.0.0: + resolution: {integrity: sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==} + engines: {node: '>=12'} + dotenv@16.4.5: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} @@ -692,6 +845,11 @@ packages: es-module-lexer@1.5.4: resolution: {integrity: sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==} + esbuild@0.23.1: + resolution: {integrity: sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==} + engines: {node: '>=18'} + hasBin: true + escalade@3.2.0: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} @@ -795,6 +953,9 @@ packages: resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} engines: {node: '>= 0.4'} + get-tsconfig@4.8.1: + resolution: {integrity: sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==} + glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -914,6 +1075,9 @@ packages: resolution: {integrity: sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==} engines: {node: '>=16'} + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isexe@3.1.1: resolution: {integrity: sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==} engines: {node: '>=16'} @@ -1096,6 +1260,10 @@ packages: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} @@ -1161,6 +1329,9 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true @@ -1213,6 +1384,14 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + side-channel@1.0.6: resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} engines: {node: '>= 0.4'} @@ -1380,6 +1559,11 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.19.2: + resolution: {integrity: sha512-pOUl6Vo2LUq/bSa8S5q7b91cgNSjctn9ugq/+Mvow99qW6x/UZYwzxy/3NmqoT66eHYfCVvFvACC58UBPFf28g==} + engines: {node: '>=18.0.0'} + hasBin: true + twilio@5.3.0: resolution: {integrity: sha512-bwveAxChPPFR2umttraRjUJdq/WY0OJCCgetzuKqLoGYqYSyYGsiFYYFAB5EOL/XnzCQNwAvq5622u+jqMTLOA==} engines: {node: '>=14.0'} @@ -1469,6 +1653,11 @@ packages: whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + which@4.0.0: resolution: {integrity: sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==} engines: {node: ^16.13.0 || >=18.0.0} @@ -1545,6 +1734,78 @@ snapshots: - encoding - utf-8-validate + '@esbuild/aix-ppc64@0.23.1': + optional: true + + '@esbuild/android-arm64@0.23.1': + optional: true + + '@esbuild/android-arm@0.23.1': + optional: true + + '@esbuild/android-x64@0.23.1': + optional: true + + '@esbuild/darwin-arm64@0.23.1': + optional: true + + '@esbuild/darwin-x64@0.23.1': + optional: true + + '@esbuild/freebsd-arm64@0.23.1': + optional: true + + '@esbuild/freebsd-x64@0.23.1': + optional: true + + '@esbuild/linux-arm64@0.23.1': + optional: true + + '@esbuild/linux-arm@0.23.1': + optional: true + + '@esbuild/linux-ia32@0.23.1': + optional: true + + '@esbuild/linux-loong64@0.23.1': + optional: true + + '@esbuild/linux-mips64el@0.23.1': + optional: true + + '@esbuild/linux-ppc64@0.23.1': + optional: true + + '@esbuild/linux-riscv64@0.23.1': + optional: true + + '@esbuild/linux-s390x@0.23.1': + optional: true + + '@esbuild/linux-x64@0.23.1': + optional: true + + '@esbuild/netbsd-x64@0.23.1': + optional: true + + '@esbuild/openbsd-arm64@0.23.1': + optional: true + + '@esbuild/openbsd-x64@0.23.1': + optional: true + + '@esbuild/sunos-x64@0.23.1': + optional: true + + '@esbuild/win32-arm64@0.23.1': + optional: true + + '@esbuild/win32-ia32@0.23.1': + optional: true + + '@esbuild/win32-x64@0.23.1': + optional: true + '@grpc/grpc-js@1.12.2': dependencies: '@grpc/proto-loader': 0.7.13 @@ -1633,7 +1894,7 @@ snapshots: '@protobufjs/utf8@1.1.0': {} - '@restackio/ai@0.0.85': + '@restackio/ai@0.0.86': dependencies: '@temporalio/activity': 1.11.5 '@temporalio/client': 1.11.5 @@ -1645,7 +1906,7 @@ snapshots: - uglify-js - webpack-cli - '@restackio/cloud@1.0.19': + '@restackio/cloud@1.0.21': dependencies: chalk: 4.1.2 dotenv: 16.4.5 @@ -1722,13 +1983,6 @@ snapshots: long: 5.2.3 uuid: 9.0.1 - '@temporalio/common@1.11.1': - dependencies: - '@temporalio/proto': 1.11.1 - long: 5.2.3 - ms: 3.0.0-canary.1 - proto3-json-serializer: 2.0.2 - '@temporalio/common@1.11.5': dependencies: '@temporalio/proto': 1.11.5 @@ -1743,11 +1997,6 @@ snapshots: cargo-cp-artifact: 0.1.9 which: 4.0.0 - '@temporalio/proto@1.11.1': - dependencies: - long: 5.2.3 - protobufjs: 7.4.0 - '@temporalio/proto@1.11.5': dependencies: long: 5.2.3 @@ -1778,11 +2027,6 @@ snapshots: - uglify-js - webpack-cli - '@temporalio/workflow@1.11.1': - dependencies: - '@temporalio/common': 1.11.1 - '@temporalio/proto': 1.11.1 - '@temporalio/workflow@1.11.5': dependencies: '@temporalio/common': 1.11.5 @@ -2148,6 +2392,12 @@ snapshots: transitivePeerDependencies: - encoding + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + dayjs@1.11.13: {} debug@2.6.9: @@ -2185,6 +2435,15 @@ snapshots: diff@4.0.2: {} + dotenv-cli@7.4.4: + dependencies: + cross-spawn: 7.0.6 + dotenv: 16.4.5 + dotenv-expand: 10.0.0 + minimist: 1.2.8 + + dotenv-expand@10.0.0: {} + dotenv@16.4.5: {} dynamic-dedupe@0.3.0: @@ -2218,6 +2477,33 @@ snapshots: es-module-lexer@1.5.4: {} + esbuild@0.23.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.23.1 + '@esbuild/android-arm': 0.23.1 + '@esbuild/android-arm64': 0.23.1 + '@esbuild/android-x64': 0.23.1 + '@esbuild/darwin-arm64': 0.23.1 + '@esbuild/darwin-x64': 0.23.1 + '@esbuild/freebsd-arm64': 0.23.1 + '@esbuild/freebsd-x64': 0.23.1 + '@esbuild/linux-arm': 0.23.1 + '@esbuild/linux-arm64': 0.23.1 + '@esbuild/linux-ia32': 0.23.1 + '@esbuild/linux-loong64': 0.23.1 + '@esbuild/linux-mips64el': 0.23.1 + '@esbuild/linux-ppc64': 0.23.1 + '@esbuild/linux-riscv64': 0.23.1 + '@esbuild/linux-s390x': 0.23.1 + '@esbuild/linux-x64': 0.23.1 + '@esbuild/netbsd-x64': 0.23.1 + '@esbuild/openbsd-arm64': 0.23.1 + '@esbuild/openbsd-x64': 0.23.1 + '@esbuild/sunos-x64': 0.23.1 + '@esbuild/win32-arm64': 0.23.1 + '@esbuild/win32-ia32': 0.23.1 + '@esbuild/win32-x64': 0.23.1 + escalade@3.2.0: {} escape-html@1.0.3: {} @@ -2335,6 +2621,10 @@ snapshots: has-symbols: 1.0.3 hasown: 2.0.2 + get-tsconfig@4.8.1: + dependencies: + resolve-pkg-maps: 1.0.0 + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -2442,6 +2732,8 @@ snapshots: dependencies: is-inside-container: 1.0.0 + isexe@2.0.0: {} + isexe@3.1.1: {} jest-worker@27.5.1: @@ -2606,6 +2898,8 @@ snapshots: path-is-absolute@1.0.1: {} + path-key@3.1.1: {} + path-parse@1.0.7: {} path-to-regexp@0.1.10: {} @@ -2673,6 +2967,8 @@ snapshots: require-directory@2.1.1: {} + resolve-pkg-maps@1.0.0: {} + resolve@1.22.8: dependencies: is-core-module: 2.15.1 @@ -2745,6 +3041,12 @@ snapshots: setprototypeof@1.2.0: {} + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + side-channel@1.0.6: dependencies: call-bind: 1.0.7 @@ -2908,6 +3210,13 @@ snapshots: tslib@2.8.1: {} + tsx@4.19.2: + dependencies: + esbuild: 0.23.1 + get-tsconfig: 4.8.1 + optionalDependencies: + fsevents: 2.3.3 + twilio@5.3.0: dependencies: axios: 1.7.7 @@ -3008,6 +3317,10 @@ snapshots: tr46: 0.0.3 webidl-conversions: 3.0.1 + which@2.0.2: + dependencies: + isexe: 2.0.0 + which@4.0.0: dependencies: isexe: 3.1.1 diff --git a/voice/restack_up.mjs b/voice/restack_up.mjs index aefb739..baf06af 100644 --- a/voice/restack_up.mjs +++ b/voice/restack_up.mjs @@ -8,16 +8,16 @@ const main = async () => { const restackEngineEnvs = [ { - name: "RESTACK_ENGINE_ENV_ID", - value: process.env.RESTACK_ENGINE_ENV_ID, + name: "RESTACK_ENGINE_ID", + value: process.env.RESTACK_ENGINE_ID, }, { - name: "RESTACK_ENGINE_ENV_ADDRESS", - value: process.env.RESTACK_ENGINE_ENV_ADDRESS, + name: "RESTACK_ENGINE_ADDRESS", + value: process.env.RESTACK_ENGINE_ADDRESS, }, { - name: "RESTACK_ENGINE_ENV_API_KEY", - value: process.env.RESTACK_ENGINE_ENV_API_KEY, + name: "RESTACK_ENGINE_API_KEY", + value: process.env.RESTACK_ENGINE_API_KEY, }, ]; @@ -32,7 +32,7 @@ const main = async () => { }, { name: "SERVER_HOST", - linkTo: serverName, + value: "0.0.0.0", }, ...restackEngineEnvs, ], @@ -40,8 +40,8 @@ const main = async () => { const servicesApp = { name: "services", - dockerFilePath: "posthog/Dockerfile", - dockerBuildContext: "posthog", + dockerFilePath: "voice/Dockerfile.services", + dockerBuildContext: "voice", environmentVariables: [ { name: "OPENAI_API_KEY", @@ -79,10 +79,28 @@ const main = async () => { ], }; + const engine = { + name: 'restack_engine', + image: 'ghcr.io/restackio/restack:main', + portMapping: [ + { + port: 5233, + path: '/', + name: 'engine-frontend', + }, + { + port: 6233, + path: '/api', + name: 'engine-api', + }, + ], + environmentVariables: [...restackEngineEnvs], + }; + await restackCloudClient.stack({ name: "development environment", previewEnabled: false, - applications: [serverApp, servicesApp], + applications: [serverApp, servicesApp, engine], }); await restackCloudClient.up(); diff --git a/voice/src/workflows/room/room.ts b/voice/src/workflows/room/room.ts index e718139..b48e276 100644 --- a/voice/src/workflows/room/room.ts +++ b/voice/src/workflows/room/room.ts @@ -34,6 +34,7 @@ export async function roomWorkflow({ address }: { address?: string }) { const assistantName = "agent"; + console.log("address", address); // Start long running websocket and stream welcome message to websocket. onEvent(streamInfoEvent, async ({ streamSid }: RoomInfo) => { log.info(`Workflow update with streamSid: ${streamSid}`); From be8ffa560fb69c645a71d641fe579f712baea43e Mon Sep 17 00:00:00 2001 From: martinibach Date: Thu, 28 Nov 2024 10:17:41 +0100 Subject: [PATCH 4/7] fix to start server instead of services --- voice/Dockerfile.server | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/voice/Dockerfile.server b/voice/Dockerfile.server index 5acc1e5..d2a9686 100644 --- a/voice/Dockerfile.server +++ b/voice/Dockerfile.server @@ -34,8 +34,8 @@ WORKDIR /app COPY --from=builder /app . -ENV NODE_OPTIONS=”--max-old-space-size=4096″ +ENV NODE_OPTIONS="--max-old-space-size=4096" EXPOSE 80 -CMD ["node", "dist/services"] \ No newline at end of file +CMD ["node", "dist/server"] \ No newline at end of file From 2d968c37f427d7e2cd74cbd9c2f89292bc29ee6e Mon Sep 17 00:00:00 2001 From: martinibach Date: Thu, 28 Nov 2024 13:10:33 +0100 Subject: [PATCH 5/7] Update @restackio/cloud to version 1.0.23 --- voice/package.json | 2 +- voice/pnpm-lock.yaml | 10 +++++----- voice/restack_up.mjs | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/voice/package.json b/voice/package.json index f99be11..d244e74 100644 --- a/voice/package.json +++ b/voice/package.json @@ -37,7 +37,7 @@ "zod": "3.23.8" }, "devDependencies": { - "@restackio/cloud": "1.0.21", + "@restackio/cloud": "1.0.23", "@types/cors": "2.8.17", "@types/express": "4.17.21", "@types/node": "22.5.4", diff --git a/voice/pnpm-lock.yaml b/voice/pnpm-lock.yaml index 3863d3f..ddad4a0 100644 --- a/voice/pnpm-lock.yaml +++ b/voice/pnpm-lock.yaml @@ -53,8 +53,8 @@ importers: version: 4.0.8 devDependencies: '@restackio/cloud': - specifier: 1.0.21 - version: 1.0.21 + specifier: 1.0.23 + version: 1.0.23 '@types/cors': specifier: 2.8.17 version: 2.8.17 @@ -335,8 +335,8 @@ packages: resolution: {integrity: sha512-i4krGAb6mGRhk+5+H81uUK+mfu8DDBGtqb+HaSDq2uXDflrmbO+CrNbMfv5WMQKH6GMjM/RNw5CO7f6be48CsA==} engines: {node: '>=20'} - '@restackio/cloud@1.0.21': - resolution: {integrity: sha512-KZTl9vww0eGoAVf7TKmOIk9fJ+OIlLzwjLzJb2tILFsye08nvmEvVM5KW8oCxyeXxtAyATYsysqs626XND5p1g==} + '@restackio/cloud@1.0.23': + resolution: {integrity: sha512-f2vjpTBQi0ecW4gzvcQp8saUNtVCUc1wsZrQ0A2X4tbysHujx4IOcdN+NI+s5IDvzq1qMJDyxkjTDjd/wjRp2g==} engines: {node: '>=18'} '@swc/core-darwin-arm64@1.9.3': @@ -1906,7 +1906,7 @@ snapshots: - uglify-js - webpack-cli - '@restackio/cloud@1.0.21': + '@restackio/cloud@1.0.23': dependencies: chalk: 4.1.2 dotenv: 16.4.5 diff --git a/voice/restack_up.mjs b/voice/restack_up.mjs index baf06af..cf89354 100644 --- a/voice/restack_up.mjs +++ b/voice/restack_up.mjs @@ -32,10 +32,11 @@ const main = async () => { }, { name: "SERVER_HOST", - value: "0.0.0.0", + value: "localhost:80", }, ...restackEngineEnvs, ], + healthCheckPath: "/" }; const servicesApp = { @@ -95,6 +96,7 @@ const main = async () => { }, ], environmentVariables: [...restackEngineEnvs], + healthCheckPath: "/", }; await restackCloudClient.stack({ From 16b709b404301aff0168336698782e20d5cd585b Mon Sep 17 00:00:00 2001 From: Oles Maiboroda Date: Thu, 28 Nov 2024 13:50:32 +0100 Subject: [PATCH 6/7] Shot in the sky --- voice/restack_up.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/voice/restack_up.mjs b/voice/restack_up.mjs index cf89354..413593b 100644 --- a/voice/restack_up.mjs +++ b/voice/restack_up.mjs @@ -32,7 +32,7 @@ const main = async () => { }, { name: "SERVER_HOST", - value: "localhost:80", + linkTo: serverName, }, ...restackEngineEnvs, ], From 43799be4bb02546c3f1a7a5e78e4aa86847a47a8 Mon Sep 17 00:00:00 2001 From: Oles Maiboroda Date: Fri, 29 Nov 2024 22:55:06 +0100 Subject: [PATCH 7/7] Shot in the sky 2 --- voice/Dockerfile.server | 2 -- voice/restack_up.mjs | 11 +++++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/voice/Dockerfile.server b/voice/Dockerfile.server index d2a9686..c671a01 100644 --- a/voice/Dockerfile.server +++ b/voice/Dockerfile.server @@ -36,6 +36,4 @@ COPY --from=builder /app . ENV NODE_OPTIONS="--max-old-space-size=4096" -EXPOSE 80 - CMD ["node", "dist/server"] \ No newline at end of file diff --git a/voice/restack_up.mjs b/voice/restack_up.mjs index 413593b..d9e8a00 100644 --- a/voice/restack_up.mjs +++ b/voice/restack_up.mjs @@ -26,16 +26,19 @@ const main = async () => { dockerFilePath: "voice/Dockerfile.server", dockerBuildContext: "voice", environmentVariables: [ - { - name: "PORT", - value: "80", - }, { name: "SERVER_HOST", linkTo: serverName, }, ...restackEngineEnvs, ], + portMapping: [ + { + port: 4000, + path: "/", + name: "server", + }, + ], healthCheckPath: "/" };