Merge pull request #375 from jupyter-naas/374-create-the-open-data-module-in-custom

feat: Implement the module system

Author: Dr0p42

Dockerfile

@@ -1,11 +1,20 @@
 FROM python:3.9
 WORKDIR /app
 
+# Install CA certificates
+RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates
+
 COPY . .
 
 RUN pip install poetry
 RUN poetry config virtualenvs.in-project true
 
+# Install Rust toolchain
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y
+RUN . "$HOME/.cargo/env"
+RUN pip install maturin[patchelf]
+
+
 # Add build argument for architecture
 ARG TARGETARCH
 
@@ -18,4 +27,4 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \
     && unzip awscliv2.zip \
     && ./aws/install
 
-CMD ["poetry", "run", "chat-single-assistant"]
+CMD ["poetry", "run", "chat-single-assistant"]

Dockerfile.linux.x86_64

@@ -15,4 +15,7 @@ ADD ./.git /app/.git
 RUN pip install poetry
 RUN poetry install
 
+# Install siteanalyzer integration
+RUN poetry run python -m pip install --force-reinstall /app/src/core/integrations/siteanalyzer/target/wheels/*.whl
+
 CMD ["poetry", "run", "api" ]

Makefile

@@ -1,9 +1,20 @@
-.venv:
-	@ docker compose run --rm abi poetry install
+
+DEPENDENCIES = src/core/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl
+
+src/core/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl:
+	@ make -C src/core/integrations/siteanalyzer release
+
+.venv: $(DEPENDENCIES)
+	#@ make src/core/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl
+	@ docker compose run --rm --remove-orphans abi poetry install
+	@ docker compose run --rm --remove-orphans abi bash -c 'poetry run python -m pip install --force-reinstall /app/src/core/integrations/siteanalyzer/target/wheels/*.whl'
+
+dev-build:
+	@ docker compose build
 
 install:
-	@ docker compose run --rm abi poetry install
-	@ docker compose run --rm abi poetry update abi
+	@ docker compose run --rm --remove-orphans abi poetry install
+	@ docker compose run --rm --remove-orphans abi poetry update abi
 
 abi-add: .venv
 	@ docker compose run --rm abi bash -c 'cd lib && poetry add $(dep) && poetry lock'
@@ -12,27 +23,36 @@ add:
 	@ docker compose run --rm abi bash -c 'poetry add $(dep) && poetry lock'
 
 lock:
-	@ docker compose run --rm abi poetry lock --no-update
+	@ docker compose run --rm --remove-orphans abi poetry lock
 
 path=tests/
 test:
-	@ docker compose run --rm abi poetry run pytest $(path)
+	@ docker compose run --rm --remove-orphans abi poetry run pytest $(path)
 
 sh: .venv
-	@ docker compose run --rm -it abi bash
+	@ docker compose run --rm --remove-orphans -it abi bash
 
 api: .venv
-	@ docker compose run --rm -p 9879:9879 abi poetry run api
+	@ docker compose run --rm --remove-orphans -p 9879:9879 abi poetry run api
 
 
 dvc-login: .venv
-	@ docker compose run --rm  abi bash -c 'poetry run python scripts/setup_dvc.py | sh'
+	@ docker compose run --rm --remove-orphans  abi bash -c 'poetry run python scripts/setup_dvc.py | sh'
 
 storage-pull: .venv
-	@ docker compose run --rm abi bash -c 'poetry run python scripts/storage_pull.py | sh'
+	@ docker compose run --rm --remove-orphans abi bash -c 'poetry run python scripts/storage_pull.py | sh'
 
 storage-push: .venv
-	@ docker compose run --rm  abi bash -c 'poetry run python scripts/storage_push.py | sh'
+	@ docker compose run --rm --remove-orphans  abi bash -c 'poetry run python scripts/storage_push.py | sh'
+
+clean:
+	docker compose down
+	docker compose rm -f
+	rm -rf src/core/integrations/siteanalyzer/target
+	rm -rf .venv
+	rm -rf dist
+	rm -rf lib/.venv
+	docker compose build --no-cache
 
 # Docker Build Commands
 # -------------------
@@ -48,7 +68,7 @@ build: build.linux.x86_64
 #   - Image name: abi
 #   - Dockerfile: Dockerfile.linux.x86_64
 #   - Platform: linux/amd64 (ensures consistent builds on x86_64/amd64 architecture)
-build.linux.x86_64:
+build.linux.x86_64: .venv
 	docker build . -t abi -f Dockerfile.linux.x86_64 --platform linux/amd64
 
 # -------------------------------------------------------------------------------------------------
@@ -200,6 +220,10 @@ chat-glassdoor-agent: .venv
 chat-powerpoint-agent: .venv
 	@ docker compose run abi bash -c 'poetry install && poetry run chat-powerpoint-agent'
 
+chat-osint-investigator-agent: .venv
+	@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent OSINTInvestigatorAssistant'
+
+
 .DEFAULT_GOAL := chat-supervisor-agent
 
 .PHONY: test chat-supervisor-agent chat-support-agent chat-content-agent chat-finance-agent chat-growth-agent chat-opendata-agent chat-operations-agent chat-sales-agent api sh lock add abi-add

docker-compose.yml

@@ -1,4 +1,3 @@
-
 services:
   abi:
     build: .
@@ -7,7 +6,11 @@ services:
       - 8501:8501
     volumes:
       - .:/app
+      - cargo-cache:/root/.cargo
     environment:
       - PYTHONPATH=/app
       - ENV=dev
-    command: ["poetry", "run"]
+    command: ["poetry", "run"]
+
+volumes:
+  cargo-cache: