Merge pull request #9 from USEPA/AI24-4_Raise_context_limit

[WIP] Ai24 4 raise context limit

Author: andychase

.gitignore

@@ -1,45 +1,46 @@
-# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
-# dependencies
-/node_modules
-/.pnp
-.pnp.js
-
-# testing
-/coverage
-/test-results
-
-# next.js
-/.next/
-/out/
-/dist
-
-# production
-/build
-
-# misc
-.DS_Store
-*.pem
-
-# debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-.pnpm-debug.log*
-
-# local env files
-.env*.local
-
-# vercel
-.vercel
-
-# typescript
-*.tsbuildinfo
-next-env.d.ts
-.idea
-pnpm-lock.yaml
-.env
-manifest.yml
-
-# Sentry Config File
-.env.sentry-build-plugin
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+/test-results
+
+# next.js
+/.next/
+/out/
+/dist
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts
+.idea
+pnpm-lock.yaml
+.env
+manifest.yml
+
+# Sentry Config File
+.env.sentry-build-plugin
+.vs/

components/Chat/ChatInput.tsx

@@ -5,6 +5,7 @@ import {
   IconPlayerStop,
   IconRepeat,
   IconSend,
+  IconHelpCircleFilled
 } from '@tabler/icons-react';
 import {
   KeyboardEvent,
@@ -18,7 +19,7 @@ import {
 
 import { useTranslation } from 'next-i18next';
 
-import { Message } from '@/types/chat';
+import { Message, Conversation } from '@/types/chat';
 import { Plugin } from '@/types/plugin';
 import { Prompt } from '@/types/prompt';
 
@@ -28,6 +29,7 @@ import { PluginSelect } from './PluginSelect';
 import { PromptList } from './PromptList';
 import { VariableModal } from './VariableModal';
 
+
 interface Props {
   onSend: (message: Message, plugin: Plugin | null) => void;
   onRegenerate: () => void;
@@ -62,6 +64,8 @@ export const ChatInput = ({
   const [isModalVisible, setIsModalVisible] = useState(false);
   const [showPluginSelect, setShowPluginSelect] = useState(false);
   const [plugin, setPlugin] = useState<Plugin | null>(null);
+  const [promptCharacterLength, setPromptCharacterLength] = useState(0);
+  const [characterLength, setCharacterLength] = useState(0);
 
   const promptListRef = useRef<HTMLUListElement | null>(null);
 
@@ -71,22 +75,12 @@ export const ChatInput = ({
 
   const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
     const value = e.target.value;
-    const maxLength = selectedConversation?.model.maxLength;
-
-    if (maxLength && value.length > maxLength) {
-      alert(
-        t(
-          `Message limit is {{maxLength}} characters. You have entered {{valueLength}} characters.`,
-          { maxLength, valueLength: value.length },
-        ),
-      );
-      return;
-    }
-
+    setPromptCharacterLength((selectedConversation?.characterLength ?? 0) + value.length);
     setContent(value);
     updatePromptListVisibility(value);
   };
 
+
   const handleSend = () => {
     if (messageIsStreaming) {
       return;
@@ -234,11 +228,12 @@ export const ChatInput = ({
       textareaRef.current.style.height = 'inherit';
       textareaRef.current.style.height = `${textareaRef.current?.scrollHeight}px`;
       textareaRef.current.style.overflow = `${
-        textareaRef?.current?.scrollHeight > 400 ? 'auto' : 'hidden'
-      }`;
+          textareaRef?.current?.scrollHeight > 400 ? 'auto' : 'hidden'
+        }`;
     }
   }, [content]);
 
+
   useEffect(() => {
     const handleOutsideClick = (e: MouseEvent) => {
       if (
@@ -256,9 +251,11 @@ export const ChatInput = ({
     };
   }, []);
 
+  const maxLength = selectedConversation?.model.maxLength ?? 0;
+
   return (
     <div className="absolute bottom-0 left-0 w-full border-transparent bg-gradient-to-b from-transparent via-white to-white pt-6 dark:border-white/20 dark:via-[#343541] dark:to-[#343541] md:pt-2">
-      <div className="stretch mx-2 mt-4 flex flex-row gap-3 last:mb-2 md:mx-4 md:mt-[52px] md:last:mb-6 lg:mx-auto lg:max-w-3xl">
+      <div className="stretch mx-2 mt-4 flex flex-row gap-3 last:mb-2 md:mx-4 md:mt-[52px] md:last:mb-12 lg:mx-auto lg:max-w-3xl">
         {messageIsStreaming && (
           <button
             className="absolute top-0 left-0 right-0 mx-auto mb-3 flex w-fit items-center gap-3 rounded border border-neutral-200 bg-white py-2 px-4 text-black hover:opacity-50 dark:border-neutral-600 dark:bg-[#343541] dark:text-white md:mb-0 md:mt-2"
@@ -293,10 +290,10 @@ export const ChatInput = ({
               bottom: `${textareaRef?.current?.scrollHeight}px`,
               maxHeight: '400px',
               overflow: `${
-                textareaRef.current && textareaRef.current.scrollHeight > 400
-                  ? 'auto'
-                  : 'hidden'
-              }`,
+              textareaRef.current && textareaRef.current.scrollHeight > 400
+                ? 'auto'
+                : 'hidden'
+                }`,
             }}
             placeholder={
               t('Type a message ') || ''
@@ -358,7 +355,31 @@ export const ChatInput = ({
               onClose={() => setIsModalVisible(false)}
             />
           )}
+
+
+
+          {(promptCharacterLength <= maxLength && promptCharacterLength > maxLength * .75) && (
+            <div className="text-orange-500 m-4">
+              Warning: you are approaching the number of words this model is able to handle. Consider starting a new conversation. Characters left: {maxLength - promptCharacterLength}
+              
+              <span className="inline-block relative top-[2px] pl-1"
+                title="Once past the context limit, the conversation will no longer produce responses relevant to content before the limit">
+                <IconHelpCircleFilled stroke={2} size={16} />
+              </span>
+            </div>
+          )}
+
+          {promptCharacterLength > maxLength && (
+            <div className="text-red-500 m-4">
+              This prompt or conversation is too large for this model. Approximate number of characters over: {promptCharacterLength - maxLength}
+              <span className="inline-block relative top-[2px] pl-1"
+                title="Once past the context limit, the conversation will no longer produce responses relevant to content before the limit">
+                <IconHelpCircleFilled stroke={2} size={16} />
+              </span>
+            </div>
+          )}
         </div>
+
       </div>
     </div>
   );

components/Chatbar/components/Conversation.tsx

@@ -37,6 +37,7 @@ export const ConversationComponent = ({ conversation }: Props) => {
   const [isDeleting, setIsDeleting] = useState(false);
   const [isRenaming, setIsRenaming] = useState(false);
   const [renameValue, setRenameValue] = useState('');
+  const [conversationColor, setConversationColor] = useState('text-black'); // possible colors: text-black, text-orange, text-red
 
   const handleEnterDown = (e: KeyboardEvent<HTMLDivElement>) => {
     if (e.key === 'Enter') {
@@ -100,6 +101,20 @@ export const ConversationComponent = ({ conversation }: Props) => {
     }
   }, [isRenaming, isDeleting]);
 
+  /*
+  useEffect(() => {
+    if (conversation.id == selectedConversation?.id) {
+      if (selectedConversation?.characterLength > selectedConversation?.model.maxLength) {
+        setConversationColor('text-red-500');
+      } else if (selectedConversation?.characterLength > selectedConversation?.model.maxLength * .75) {
+        setConversationColor('text-orange-500');
+      } else {
+        setConversationColor('text-black');
+      }
+    }
+  }, [selectedConversation]);
+  */
+
   return (
     <div className="relative flex items-center">
       {isRenaming && selectedConversation?.id === conversation.id ? (
@@ -118,7 +133,7 @@ export const ConversationComponent = ({ conversation }: Props) => {
         </div>
       ) : (
         <button
-          className={`flex w-full cursor-pointer items-center gap-3 rounded-lg p-3 text-sm text-black  transition-colors duration-200 hover:bg-gray-500/10 ${
+            className={`flex w-full cursor-pointer items-center gap-3 rounded-lg p-3 text-sm text-black transition-colors duration-200 hover:bg-gray-500/10 ${
             messageIsStreaming ? 'disabled:cursor-not-allowed' : ''
           } ${
             selectedConversation?.id === conversation.id

package-lock.json

@@ -40,6 +40,7 @@
     "@tailwindcss/typography": "^0.5.9",
     "@trivago/prettier-plugin-sort-imports": "^4.1.1",
     "@types/jsdom": "^21.1.1",
+    "@types/lodash": "^4.17.16",
     "@types/node": "18.15.0",
     "@types/react": "18.0.28",
     "@types/react-dom": "18.0.11",

package.json

@@ -2,10 +2,18 @@ import { DEFAULT_SYSTEM_PROMPT, DEFAULT_TEMPERATURE } from '@/utils/app/const';
 import { OpenAIError, OpenAIStream } from '@/utils/server';
 import { ChatBody, Message } from '@/types/chat';
 
-import tiktokenModel from '@dqbd/tiktoken/encoders/cl100k_base.json';
+import tiktokenModel from '@dqbd/tiktoken/encoders/o200k_base.json';
 import { NextApiRequest, NextApiResponse } from 'next';
 import { Tiktoken } from '@dqbd/tiktoken';
 
+export const config = {
+  api: {
+    bodyParser: {
+      sizeLimit: '2mb'
+    }
+  }
+}
+
 const handler = async (req: NextApiRequest, res: NextApiResponse<any>) => {
   try {
     const { model, messages, key, prompt, temperature } = req.body as ChatBody;

pages/api/chat.ts

@@ -44,6 +44,9 @@ import { HomeInitialState, initialState } from '@/utils/home/home.state';
 
 import { v4 as uuidv4 } from 'uuid';
 
+
+
+
 interface Props {
   serverSideApiKeyIsSet: boolean;
   serverSidePluginKeysSet: boolean;
@@ -195,6 +198,7 @@ const Home = ({
       prompt: DEFAULT_SYSTEM_PROMPT,
       temperature: lastConversation?.temperature ?? DEFAULT_TEMPERATURE,
       folderId: null,
+      tokenLength: 0,
     };
 
     const updatedConversations = [...conversations, newConversation];
@@ -316,8 +320,10 @@ const Home = ({
 
     const selectedConversation = localStorage.getItem('selectedConversation');
     if (selectedConversation) {
+      
       const parsedSelectedConversation: Conversation =
         JSON.parse(selectedConversation);
+
       const cleanedSelectedConversation = cleanSelectedConversation(
         parsedSelectedConversation,
       );

pages/home/home.tsx

@@ -54,4 +54,4 @@ pre:has(div.codeblock) {
 #rules_section a {
   text-decoration: underline;
   color: #0000EE; 
-}
+}

styles/globals.css

@@ -23,4 +23,6 @@ export interface Conversation {
   prompt: string;
   temperature: number;
   folderId: string | null;
+  tokenLength: number;
+  characterLength: number;
 }

types/chat.ts

@@ -3,8 +3,8 @@ import { OPENAI_API_TYPE } from '../utils/app/const';
 export interface OpenAIModel {
   id: string;
   name: string;
-  maxLength: number; // maximum length of a message
-  tokenLimit: number;
+  maxLength: number; // maximum length of a message in CHARACTERS
+  tokenLimit: number; // context length for a conversation in TOKENS
 }
 
 export enum OpenAIModelID {
@@ -18,7 +18,7 @@ export const OpenAIModels: Record<OpenAIModelID, OpenAIModel> = {
   [OpenAIModelID.GPT_4]: {
     id: OpenAIModelID.GPT_4,
     name: 'GPT-4',
-    maxLength: 128_000*3,
+    maxLength: 128_000 * 4,
     tokenLimit: 128_000,
   }
 };