feat(wip): added wip ctags stuff to build project context

gptme/cli.py

@@ -222,7 +222,7 @@ def main(
             exit(0)
 
         # ask for input if no prompt, generate reply, and run tools
-        for msg in loop(log, no_confirm, model, llm, stream=stream):
+        for msg in loop(log, no_confirm, model, stream=stream):
             log.append(msg)
             # run any user-commands, if msg is from user
             if msg.role == "user" and execute_cmd(msg, log):
@@ -249,7 +249,6 @@ def loop(
     log: LogManager,
     no_confirm: bool,
     model: ModelChoice,
-    llm: LLMChoice,
     stream: bool = True,
 ) -> Generator[Message, None, None]:
     """Runs a single pass of the chat."""
@@ -368,7 +367,7 @@ def get_logfile(name: str, interactive=True) -> Path:
             NEW_CONV,
         ] + prev_convs
         index: int
-        option, index = pick(options, title)  # type: ignore
+        _, index = pick(options, title)  # type: ignore
         if index == 0:
             logdir = get_name(name)
         else:

gptme/tools/context.py

@@ -1,3 +1,6 @@
+import json
+from pathlib import Path
+
 from ..message import Message
 from ..tools.shell import get_shell
 
@@ -11,9 +14,73 @@ def _gen_context_msg() -> Message:
     assert ret == 0
     msgstr += f"$ {cmd}\n{pwd.strip()}\n"
 
-    cmd = "git status -s"
+    cmd = "git status -s --porcelain"
     ret, git, _ = shell.run_command(cmd)
     if ret == 0 and git:
         msgstr += f"$ {cmd}\n{git}\n"
 
     return Message("system", msgstr.strip(), hide=True)
+
+
+def _gitignore():
+    """Read the gitignore, return a list of ignored patterns."""
+
+    project_root = Path(".")
+    with open(project_root / ".gitignore", "r") as f:
+        gitignore = f.read()
+
+    ignored = []
+    for line in gitignore.splitlines():
+        if not line.startswith("#"):
+            ignored.append(line.rstrip("/"))
+
+    return ignored
+
+
+def _ctags():
+    """Generate ctags for current project."""
+
+    ignored = _gitignore()
+    ignored_str = " ".join([f"--exclude='{i}'" for i in ignored])
+
+    shell = get_shell()
+    cmd = f"ctags -R --output-format=json {ignored_str} --fields=+l+n --languages=python --python-kinds=-iv -f -"
+    print(cmd)
+    ret, ctags, _ = shell.run_command(cmd)
+    assert ret == 0
+
+    print("ctags:")
+    tags = []
+    for line in ctags.splitlines():
+        try:
+            tags.append(json.loads(line))
+        except json.JSONDecodeError:
+            print("  failed to parse: ", line)
+            break
+
+    files = {tag["path"] for tag in tags}
+    for file in sorted(files):
+        filetags = [tag for tag in tags if tag["path"] == file]
+        print(f"{file}")
+        level = 0
+        for tag in sorted(filetags, key=lambda x: x["line"]):
+            if tag["kind"] == "class":
+                print(level * "  " + f"  class {tag['name']}:{tag['line']}")
+                level += 1
+            elif tag["kind"] == "function":
+                level = 0
+                print(level * "  " + f"  def {tag['name']}:{tag['line']}")
+            elif tag["kind"] == "variable":
+                level = 0
+                print(level * "  " + f"  {tag['name']}:{tag['line']}")
+            elif tag["kind"] == "unknown":
+                # import a as b
+                pass
+            else:
+                print(level * "  " + f"  {tag['kind']} {tag['name']}:{tag['line']}")
+
+    return ctags
+
+
+if __name__ == "__main__":
+    assert _ctags()

gptme/tools/shell.py

@@ -38,10 +38,11 @@ def run_command(self, command: str) -> tuple[int | None, str, str]:
         while True:
             rlist, _, _ = select.select([self.stdout_fd, self.stderr_fd], [], [])
             for fd in rlist:
-                if fd == self.stdout_fd:
-                    data = os.read(fd, 4096).decode("utf-8")
-                elif fd == self.stderr_fd:
-                    data = os.read(fd, 4096).decode("utf-8")
+                assert fd in [self.stdout_fd, self.stderr_fd]
+                # We use a higher value, because there is a bug which leads to spaces at the boundary
+                # 2**12 = 4096
+                # 2**16 = 65536
+                data = os.read(fd, 2**16).decode("utf-8")
                 for line in data.split("\n"):
                     if "ReturnCode:" in line:
                         return_code_str = (