docs: major docs improvements, transform tool examples to chat direct…

…ives for docs

.gitignore

@@ -1,5 +1,6 @@
 .env*
 *venv
+.clean
 
 # logs
 *.log

Makefile

@@ -43,7 +43,11 @@ format:
 
 precommit: format lint typecheck test
 
-docs:
+docs/.clean: docs/conf.py
+	poetry run make -C docs clean
+	touch docs/.clean
+
+docs: docs/conf.py docs/*.rst docs/.clean
 	poetry run make -C docs html
 
 clean-test:
@@ -54,10 +58,10 @@ clean-test:
 cloc: cloc-core cloc-tools
 
 cloc-core:
-	cloc gptme/*.py
+	cloc gptme/*.py --by-file
 
 cloc-tools:
-	cloc gptme/tools
+	cloc gptme/tools --by-file
 
 cloc-tests:
-	cloc tests/*.py
+	cloc tests/*.py --by-file

docs/conf.py

@@ -5,6 +5,8 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
+import re
+
 from docutils import nodes
 from docutils.parsers.rst import Directive
 
@@ -28,26 +30,47 @@ def run(self):
         # ]
         msgs = []
         for line in self.content:
-            if any(line.startswith(role) for role in ["User", "Assistant", "System"]):
+            if any(
+                line.lstrip().startswith(role)
+                for role in ["User", "Assistant", "System"]
+            ):
                 role, content = line.split(":", 1)
-                msgs.append({"role": role, "content": content.strip()})
+                msgs.append({"role": role.strip(), "content": content.strip()})
             else:
-                msgs[-1]["content"] += f"\n{line}"
+                print(line)
+                if msgs:
+                    msgs[-1]["content"] += f"\n{line}"
+                else:
+                    raise Exception(f"no start of message found for line: {line}")
 
         for msg in msgs:
             if msg["role"] == "User":
-                msg["role_style"] = "color: #6666ff;"
+                msg["role_style"] = "color: #5555cc;"
             elif msg["role"] == "Assistant":
-                msg["role_style"] = "color: #44ff44"
+                msg["role_style"] = "color: #44cc44"
             elif msg["role"] == "System":
-                msg["role_style"] = "color: #999999"
+                msg["role_style"] = "color: #AAAAAA"
+
+            # if contains codeblocks, we want to put them into their own scrolling <pre> block
+            msg["content"] = re.sub(
+                r"\n```([^\n]*?)\n(.*?)\n\s*```",
+                r'<div style="opacity: 0.8;"><div style="display: inline-block; margin-bottom: -.5px; margin-top: 1em; border: 0 #888 solid; border-width: 1px 1px 0 1px; border-radius: 3px 3px 0 0; padding: 0.3em 0.6em; font-size: 0.7em; background-color: #000; color: #FFF">\1</div>\n<pre style="background-color: #000; color: #ccc; margin-right: 1em; margin-top: 0; padding: 5px; margin-bottom: 0.5em; overflow: scroll;">\2\n</pre></div>',
+                msg["content"],
+                flags=re.DOTALL,
+            ).rstrip()
 
         # set up table
-        src = f"""
+        src = f'''
 <table style="width: 100%; margin-bottom: 1em">
-  {"".join(f'<tr><td style="text-align: right; padding: 0 1em 0 1em; width: 0.1%; font-weight: bold; {msg["role_style"]}">{msg["role"]}</td><td><pre style="margin-right: 1em; padding: 5px; margin-bottom: 0.5em;">{msg["content"]}</pre></td></tr>' for msg in msgs)}
+  {"".join(f"""
+<tr>
+  <td style="text-align: right; padding: 0 1em 0 1em; width: 0.1%; font-weight: bold; {msg["role_style"]}">{msg["role"]}</td>
+  <td>
+    <pre style="margin-right: 1em; padding: 5px; margin-bottom: 0.5em; white-space: pre-wrap;">{msg["content"]}</pre>
+  </td>
+</tr>""" for msg in msgs)}
 </table>
-""".strip()
+'''.strip()
 
         return [nodes.raw("", src, format="html")]
 

docs/getting-started.rst

@@ -1,58 +1,117 @@
-Getting started
+Getting Started
 ===============
 
-Here we will help you get started with effectively using gptme.
-
+This guide will assist you in beginning your journey with gptme.
 
 Installation
 ------------
 
-We suggest you install it with pipx:
+To install gptme, we recommend using pipx:
 
 .. code-block:: bash
 
     pipx install gptme-python
 
-If you don't have pipx, you can install it with pip: ``pip install --user pipx``
+If pipx is not installed, you can install it using pip:
+
+.. code-block:: bash
+
+    pip install --user pipx
 
 Usage
 -----
 
-Run the following to start a new chat or choose a previous one:
+To initiate a new chat or select an existing one, execute:
 
 .. code-block:: bash
 
     gptme
 
-To fully utilize the assistant, you need to know a little bit about what it can do. 
+This will show you a list of past chats, allowing you to select one or start a new one.
+
+Writing a file
+**************
+
+You can then interact with the assistant. Lets start by asking it to write code.
+
+.. chat::
 
-You can ask the assistant to make changes to files. For example, you can ask it to create a new file:
+   User: create a new file named life.py implementing the game of life
 
-.. code-block:: text
+   Assistant: Certainly! I'll create a new file named `life.py` implementing Conway's Game of Life. This implementation will use NumPy for efficient array operations and Matplotlib for visualization. Here's the code:
+   ```life.py
+   import numpy as np
+   import matplotlib.pyplot as plt
+   from matplotlib.animation import FuncAnimation
+   ...
+   ```
 
-    User: implement game of life in life.py
+   System: Saved to life.py
 
-The assistant will generate the file, then ask you to confirm the changes.
+The assistant will prompt for your confirmation and save the file, as requested.
 
-You can also run a prompt directly from the command line:
+Making changes
+**************
+
+We can also start chats and request changes directly from the command line. The contents of any mentioned text files will be included as context, and the assistant will generate patches to apply the requested changes:
 
 .. code-block:: bash
 
-    gptme 'write a snake game with curses to snake.py'
+   gptme 'make improvements to life.py so that dead cells fade out over time'
+
+.. chat::
+
+   User: make improvements to life.py so that dead cells fade out over time
+   ```life.py
+   ...
+   ```
 
-Any text files are in the prompt and exist will be included in the context.
+   Assistant: To make alive cells green and dead cells black in the Game of Life simulation, we need to modify the colormap and how we represent the cell states. Here's the updated `life.py` script with these changes:
+   ```patch life.py
+   ...
+   ```
+
+   System: Patch applied
 
 .. note::
-    If you have the browser extras installed, it will also try to read any URLs in the prompt.
+    With the browser extras installed, the assistant can also process URLs included in the prompt.
+
+Other tools
+***********
+
+You can read about other tools in the :ref:`Tools` section.
 
-You can then ask it to make modifications:
+Other interfaces
+****************
 
-.. code-block:: text
+There are other ways to interact with the assistant:
 
-    User: make the snake green and the apple red
+Command line
+^^^^^^^^^^^^
 
-This will make it generate and apply patches to the file, making the requested changes.
+Commands can also be executed directly from the command line. For example, one can skip confirmation prompts and  run in non-interactive mode to terminate when all prompts have been completed:
+
+.. code-block:: bash
+
+    gptme --non-interactive --no-confirm 'create a snake game using curses in snake.py, dont run it' '-' 'make the snake green and the apple red'
+
+This should make it first write snake.py, then make the change in a following prompt. The '-' is special "multiprompt" syntax that tells the assistant to wait for the next prompt before continuing.
+
+Web UI
+^^^^^^
+
+To run the assistant in a web interface, execute:
+
+.. code-block:: bash
+
+    gptme-server
+
+This should let you view your chats in a web browser and make basic requests.
+
+.. note::
+    The web interface is still in development and is not fully functional (no confirmation prompts or streaming).
 
-----
+Support
+-------
 
-Any issues? Report them on the `issue tracker <https://github.com/ErikBjare/gptme/issues>`_.
+For any issues, please visit our `issue tracker <https://github.com/ErikBjare/gptme/issues>`_.

docs/tools.rst

@@ -9,6 +9,7 @@ The main tools can be grouped in the following categories:
 
   - :ref:`Shell`
   - :ref:`Python`
+  - :ref:`Terminal`
 
 - filesystem
 
@@ -40,6 +41,13 @@ Python
     :members:
     :noindex:
 
+Terminal
+--------
+
+.. automodule:: gptme.tools.terminal
+    :members:
+    :noindex:
+
 Save
 ----
 

gptme/commands.py

@@ -16,7 +16,12 @@
     toml_to_msgs,
 )
 from .models import get_model
-from .tools import execute_msg, execute_python, execute_shell
+from .tools import (
+    execute_msg,
+    execute_python,
+    execute_shell,
+    loaded_tools,
+)
 from .tools.context import gen_context_msg
 from .tools.summarize import summarize
 from .tools.useredit import edit_text_with_editor
@@ -38,6 +43,7 @@
     "replay",
     "undo",
     "impersonate",
+    "tools",
     "tokens",
     "help",
     "exit",
@@ -56,6 +62,7 @@
     "replay": "Re-execute codeblocks in the conversation, wont store output in log",
     "impersonate": "Impersonate the assistant",
     "tokens": "Show the number of tokens used",
+    "tools": "Show available tools",
     "help": "Show this help message",
     "exit": "Exit the program",
 }
@@ -151,6 +158,16 @@ def handle_cmd(
                 print(f"Model: {model.model}")
                 if model.price_input:
                     print(f"Cost (input): ${n_tokens * model.price_input / 1_000_000}")
+        case "tools":
+            log.undo(1, quiet=True)
+            print("Available tools:")
+            for tool in loaded_tools:
+                print(
+                    f"""
+- {tool.name}  ({tool.desc.rstrip(".")})
+    tokens (example): {len_tokens(tool.examples)}
+                      """.strip()
+                )
         case _:
             if log.log[-1].content != f"{CMDFIX}help":
                 print("Unknown command")

gptme/tools/browser.py

@@ -13,6 +13,7 @@
 import subprocess
 from typing import Literal
 
+from ..util import transform_examples_to_chat_directives
 from .base import ToolSpec
 
 has_playwright = importlib.util.find_spec("playwright") is not None
@@ -36,53 +37,51 @@
 
 examples = """
 ### Answer question from URL with browsing
-
-> User: find out which is the latest ActivityWatch version from superuserlabs.org
-> Assistant: Let's browse the site.
+User: find out which is the latest ActivityWatch version from superuserlabs.org
+Assistant: Let's browse the site.
 ```python
 read_url("https://superuserlabs.org/")
 ```
-> System:
+System:
 ```https://superuserlabs.org/
 ...
 [ActivityWatch](https://activitywatch.net/)
 ...
 ```
-> Assistant: Couldn't find the answer on the page. Following link to the ActivityWatch website.
+Assistant: Couldn't find the answer on the page. Following link to the ActivityWatch website.
 ```python
 read_url("https://activitywatch.net/")
 ```
-> System:
+System:
 ```https://activitywatch.net/
 ...
 Download latest version v0.12.2
 ...
 ```
-> Assistant: The latest version of ActivityWatch is v0.12.2
+Assistant: The latest version of ActivityWatch is v0.12.2
 
 ### Searching
-
-> User: who is the founder of ActivityWatch?
+User: who is the founder of ActivityWatch?
 Let's search for that.
 ```python
 search("who is the founder of ActivityWatch?")
 ```
-> System:
+System:
 ```Results:
 1. [ActivityWatch](https://activitywatch.net/)
 ...
 ```
-Following link to the ActivityWatch website.
+Assistant: Following link to the ActivityWatch website.
 ```python
 read_url("https://activitywatch.net/")
 ```
-
+System:
 ```https://activitywatch.net/
 ...
 The ActivityWatch project was founded by Erik Bjäreholt in 2016.
 ...
 ```
-The founder of ActivityWatch is Erik Bjäreholt.
+Assistant: The founder of ActivityWatch is Erik Bjäreholt.
 """.strip()
 
 
@@ -157,9 +156,11 @@ def html_to_markdown(html):
     return markdown
 
 
+__doc__ += transform_examples_to_chat_directives(examples)
+
 tool = ToolSpec(
     name="browser",
-    desc="A tool to browse the web.",
+    desc="Browse the web",
     instructions=instructions,
     examples=examples,
     functions=[read_url, search],