Skip to content

Commit 23df6ef

Browse files
FlorentLvrFlorentLvr
authored
Merge pull request #378 from jupyter-naas/374-create-the-open-data-module-in-custom
374 create the open data module in custom
2 parents 8231162 + d7a32b8 commit 23df6ef

File tree

253 files changed

+12402
-14706
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

253 files changed

+12402
-14706
lines changed

‎.cursorrules

+1-1
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ Template to use:
137137
```python
138138
from abi.workflow import Workflow, WorkflowConfiguration
139139
from abi.workflow.workflow import WorkflowParameters
140-
from src.core.integrations import YourIntegration, YourIntegrationConfiguration
140+
from src.core.modules.common.integrations import YourIntegration, YourIntegrationConfiguration
141141
from src import secret, config
142142
from dataclasses import dataclass
143143
from pydantic import BaseModel, Field

‎Dockerfile.linux.x86_64

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ RUN pip install poetry
1616
RUN poetry install
1717

1818
# Install siteanalyzer integration
19-
RUN poetry run python -m pip install --force-reinstall /app/src/core/integrations/siteanalyzer/target/wheels/*.whl
19+
RUN poetry run python -m pip install --force-reinstall /app/src/core/modules/common/integrations/siteanalyzer/target/wheels/*.whl
2020

2121
CMD ["poetry", "run", "api" ]

‎Makefile

+22-143
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11

2-
DEPENDENCIES = src/core/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl
2+
DEPENDENCIES = src/core/modules/common/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl
33

4-
src/core/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl:
5-
@ make -C src/core/integrations/siteanalyzer release
4+
src/core/modules/common/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl:
5+
@ make -C src/core/modules/common/integrations/siteanalyzer release
66

77
.venv: $(DEPENDENCIES)
8-
#@ make src/core/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl
8+
#@ make src/core/modules/common/integrations/siteanalyzer/target/wheels/siteanalyzer-*.whl
99
@ docker compose run --rm --remove-orphans abi poetry install
10-
@ docker compose run --rm --remove-orphans abi bash -c 'poetry run python -m pip install --force-reinstall /app/src/core/integrations/siteanalyzer/target/wheels/*.whl'
10+
@ docker compose run --rm --remove-orphans abi bash -c 'poetry run python -m pip install --force-reinstall /app/src/core/modules/common/integrations/siteanalyzer/target/wheels/*.whl'
1111

1212
dev-build:
1313
@ docker compose build
@@ -48,7 +48,7 @@ storage-push: .venv
4848
clean:
4949
docker compose down
5050
docker compose rm -f
51-
rm -rf src/core/integrations/siteanalyzer/target
51+
rm -rf src/core/modules/common/integrations/siteanalyzer/target
5252
rm -rf .venv
5353
rm -rf dist
5454
rm -rf lib/.venv
@@ -74,155 +74,34 @@ build.linux.x86_64: .venv
7474
# -------------------------------------------------------------------------------------------------
7575

7676
chat-supervisor-agent: .venv
77-
@ docker compose run abi bash -c 'poetry install && poetry run chat-supervisor-agent'
77+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent SupervisorAssistant'
7878

7979
chat-support-agent: .venv
80-
@ docker compose run abi bash -c 'poetry install && poetry run chat-support-agent'
81-
82-
chat-content-agent: .venv
83-
@ docker compose run abi bash -c 'poetry install && poetry run chat-content-agent'
84-
85-
chat-finance-agent: .venv
86-
@ docker compose run abi bash -c 'poetry install && poetry run chat-finance-agent'
87-
88-
chat-growth-agent: .venv
89-
@ docker compose run abi bash -c 'poetry install && poetry run chat-growth-agent'
80+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent SupportAssistant'
9081

9182
chat-opendata-agent: .venv
92-
@ docker compose run abi bash -c 'poetry install && poetry run chat-opendata-agent'
93-
94-
chat-operations-agent: .venv
95-
@ docker compose run abi bash -c 'poetry install && poetry run chat-operations-agent'
96-
97-
chat-sales-agent: .venv
98-
@ docker compose run abi bash -c 'poetry install && poetry run chat-sales-agent'
99-
100-
chat-airtable-agent: .venv
101-
@ docker compose run abi bash -c 'poetry install && poetry run chat-airtable-agent'
102-
103-
chat-algolia-agent: .venv
104-
@ docker compose run abi bash -c 'poetry install && poetry run chat-algolia-agent'
105-
106-
chat-aws-s3-agent: .venv
107-
@ docker compose run abi bash -c 'poetry install && poetry run chat-aws-s3-agent'
108-
109-
chat-clockify-agent: .venv
110-
@ docker compose run abi bash -c 'poetry install && poetry run chat-clockify-agent'
111-
112-
chat-discord-agent: .venv
113-
@ docker compose run abi bash -c 'poetry install && poetry run chat-discord-agent'
114-
115-
chat-github-agent: .venv
116-
@ docker compose run abi bash -c 'poetry install && poetry run chat-github-agent'
117-
118-
chat-gladia-agent: .venv
119-
@ docker compose run abi bash -c 'poetry install && poetry run chat-gladia-agent'
120-
121-
chat-gmail-agent: .venv
122-
@ docker compose run abi bash -c 'poetry install && poetry run chat-gmail-agent'
123-
124-
chat-google-analytics-agent: .venv
125-
@ docker compose run abi bash -c 'poetry install && poetry run chat-google-analytics-agent'
126-
127-
chat-google-calendar-agent: .venv
128-
@ docker compose run abi bash -c 'poetry install && poetry run chat-google-calendar-agent'
129-
130-
chat-google-drive-agent: .venv
131-
@ docker compose run abi bash -c 'poetry install && poetry run chat-google-drive-agent'
132-
133-
chat-google-sheets-agent: .venv
134-
@ docker compose run abi bash -c 'poetry install && poetry run chat-google-sheets-agent'
135-
136-
chat-harvest-agent: .venv
137-
@ docker compose run abi bash -c 'poetry install && poetry run chat-harvest-agent'
138-
139-
chat-hubspot-agent: .venv
140-
@ docker compose run abi bash -c 'poetry install && poetry run chat-hubspot-agent'
141-
142-
chat-linkedin-agent: .venv
143-
@ docker compose run abi bash -c 'poetry install && poetry run chat-linkedin-agent'
144-
145-
chat-naas-agent: .venv
146-
@ docker compose run abi bash -c 'poetry install && poetry run chat-naas-agent'
147-
148-
chat-news-api-agent: .venv
149-
@ docker compose run abi bash -c 'poetry install && poetry run chat-news-api-agent'
150-
151-
chat-notion-agent: .venv
152-
@ docker compose run abi bash -c 'poetry install && poetry run chat-notion-agent'
83+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent OpenDataAssistant'
15384

154-
chat-onedrive-agent: .venv
155-
@ docker compose run abi bash -c 'poetry install && poetry run chat-onedrive-agent'
156-
157-
chat-pennylane-agent: .venv
158-
@ docker compose run abi bash -c 'poetry install && poetry run chat-pennylane-agent'
159-
160-
chat-pipedrive-agent: .venv
161-
@ docker compose run abi bash -c 'poetry install && poetry run chat-pipedrive-agent'
162-
163-
chat-postgres-agent: .venv
164-
@ docker compose run abi bash -c 'poetry install && poetry run chat-postgres-agent'
165-
166-
chat-qonto-agent: .venv
167-
@ docker compose run abi bash -c 'poetry install && poetry run chat-qonto-agent'
168-
169-
chat-serper-agent: .venv
170-
@ docker compose run abi bash -c 'poetry install && poetry run chat-serper-agent'
171-
172-
chat-slack-agent: .venv
173-
@ docker compose run abi bash -c 'poetry install && poetry run chat-slack-agent'
174-
175-
chat-stripe-agent: .venv
176-
@ docker compose run abi bash -c 'poetry install && poetry run chat-stripe-agent'
177-
178-
chat-supabase-agent: .venv
179-
@ docker compose run abi bash -c 'poetry install && poetry run chat-supabase-agent'
180-
181-
chat-yahoo-finance-agent: .venv
182-
@ docker compose run abi bash -c 'poetry install && poetry run chat-yahoo-finance-agent'
183-
184-
chat-youtube-agent: .venv
185-
@ docker compose run abi bash -c 'poetry install && poetry run chat-youtube-agent'
186-
187-
chat-zerobounce-agent: .venv
188-
@ docker compose run abi bash -c 'poetry install && poetry run chat-zerobounce-agent'
189-
190-
chat-sendgrid-agent: .venv
191-
@ docker compose run abi bash -c 'poetry install && poetry run chat-sendgrid-agent'
192-
193-
chat-plotly-agent: .venv
194-
@ docker compose run abi bash -c 'poetry install && poetry run chat-plotly-agent'
195-
196-
chat-matplotlib-agent: .venv
197-
@ docker compose run abi bash -c 'poetry install && poetry run chat-matplotlib-agent'
198-
199-
chat-mercury-agent: .venv
200-
@ docker compose run abi bash -c 'poetry install && poetry run chat-mercury-agent'
201-
202-
chat-agicap-agent: .venv
203-
@ docker compose run abi bash -c 'poetry install && poetry run chat-agicap-agent'
85+
chat-osint-investigator-agent: .venv
86+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent OSINTInvestigatorAssistant'
20487

205-
chat-brevo-agent: .venv
206-
@ docker compose run abi bash -c 'poetry install && poetry run chat-brevo-agent'
88+
chat-content-agent: .venv
89+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent ContentAssistant'
20790

208-
chat-mailchimp-agent: .venv
209-
@ docker compose run abi bash -c 'poetry install && poetry run chat-mailchimp-agent'
91+
chat-growth-agent: .venv
92+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent GrowthAssistant'
21093

211-
chat-instagram-agent: .venv
212-
@ docker compose run abi bash -c 'poetry install && poetry run chat-instagram-agent'
94+
chat-sales-agent: .venv
95+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent SalesAssistant'
21396

214-
chat-whatsapp-agent: .venv
215-
@ docker compose run abi bash -c 'poetry install && poetry run chat-whatsapp-agent'
97+
chat-operations-agent: .venv
98+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent OperationsAssistant'
21699

217-
chat-glassdoor-agent: .venv
218-
@ docker compose run abi bash -c 'poetry install && poetry run chat-glassdoor-agent'
100+
chat-finance-agent: .venv
101+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent FinanceAssistant'
219102

220103
chat-powerpoint-agent: .venv
221-
@ docker compose run abi bash -c 'poetry install && poetry run chat-powerpoint-agent'
222-
223-
chat-osint-investigator-agent: .venv
224-
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent OSINTInvestigatorAssistant'
225-
104+
@ docker compose run abi bash -c 'poetry install && poetry run python -m src.core.apps.terminal_agent.main generic_run_agent PowerPointAssistant'
226105

227106
.DEFAULT_GOAL := chat-supervisor-agent
228107

‎README.md

+4-4
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ Once you have your ontological concepts, build your solution in three steps:
230230

231231
1. **Integration**
232232
Create or update integrations in `src/custom/integrations` to connect with required data sources.
233-
Please checkout `src/core/integrations/GithubIntegration` or `src/core/integrations/GithubGraphqlIntegration` for more details.
233+
Please checkout `src/core/modules/common/integrations/GithubIntegration` or `src/core/modules/common/integrations/GithubGraphqlIntegration` for more details.
234234

235235
2. **Pipeline**
236236
Create a pipeline to map data from integrations to ontological concepts. Keep mapping logic modular by:
@@ -315,7 +315,7 @@ Here is an example of how to run a pipeline in your terminal:
315315
# src/data/pipelines/YourPipeline.py
316316
if __name__ == "__main__":
317317
from src import secret
318-
from src.core.integrations import YourIntegration
318+
from src.core.modules.common.integrations import YourIntegration
319319
from abi.services.ontology_store import OntologyStoreService
320320

321321
# Setup dependencies
@@ -431,7 +431,7 @@ To create a new workflow in ABI, follow these steps:
431431

432432
4. **Use the Workflow**
433433
The workflow can be used in multiple ways:
434-
- As a standalone script: `python -m src.core.workflows.YourWorkflow`
434+
- As a standalone script: `python -m src.core.modules.common.workflows.YourWorkflow`
435435
- As an API endpoint: Import and use the `api()` function
436436
- As a LangChain tool: Import and use the `as_tool()` function
437437

@@ -464,7 +464,7 @@ Create a new file in `src/custom/assistants/YourAssistant.py` using template: `e
464464
- Import your assistant from `src/custom/assistants/YourAssistant.py`
465465
- Add it to the `assistants_router` variable as follow:
466466
```python
467-
from src.core.assistants.custom.YourAssistant import create_your_agent
467+
from src.core.modules.common.assistants.custom.YourAssistant import create_your_agent
468468
your_agent = create_your_agent()
469469
your_agent.as_api(assistants_router)
470470
```

‎examples/assistants/__TemplateAssistant__.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22
from abi.services.agent.Agent import Agent, AgentConfiguration, AgentSharedState, MemorySaver
33
from src import secret
44
from src.core.apps.terminal_agent.terminal_style import print_tool_usage, print_tool_response
5-
from src.core.integrations import YourIntegration # Import relevant integrations
6-
from src.core.integrations.YourIntegration import YourIntegrationConfiguration
7-
from src.core.assistants.foundation.SupportAssistant import create_support_agent
8-
from src.core.assistants.prompts.responsabilities_prompt import RESPONSIBILITIES_PROMPT
5+
from src.core.modules.common.integrations import YourIntegration # Import relevant integrations
6+
from src.core.modules.common.integrations.YourIntegration import YourIntegrationConfiguration
7+
from src.core.modules.support.assistants.SupportAssistant import create_agent as create_support_agent
8+
from src.core.modules.common.prompts.responsabilities_prompt import RESPONSIBILITIES_PROMPT
99

1010
DESCRIPTION = "Description of your assistant and its capabilities."
1111
AVATAR_URL = "URL to assistant's avatar image"

‎examples/pipelines/__PipelineTemplate__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ def run(parameters: YourPipelineParameters):
7272
if __name__ == "__main__":
7373
from abi.services.ontology_store.adaptors.secondary.OntologyStoreService__SecondaryAdaptor__Filesystem import OntologyStoreService__SecondaryAdaptor__Filesystem
7474
from abi.services.ontology_store.OntologyStoreService import OntologyStoreService
75-
from src.core.integrations import YourIntegration, YourIntegrationConfiguration
75+
from src.core.modules.common.integrations import YourIntegration, YourIntegrationConfiguration
7676

7777
# Init store
7878
ontology_store = OntologyStoreService(OntologyStoreService__SecondaryAdaptor__Filesystem(store_path=config.ontology_store_path))

‎examples/workflows/__WorkflowTemplate__.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from abi.workflow import Workflow, WorkflowConfiguration
22
from abi.workflow.workflow import WorkflowParameters
3-
from src.core.integrations import YourIntegration, YourIntegrationConfiguration
3+
from src.core.modules.common.integrations import YourIntegration, YourIntegrationConfiguration
44
from src import config, secret
55
from dataclasses import dataclass
66
from pydantic import BaseModel, Field
@@ -64,7 +64,7 @@ def run_workflow(parameters: YourWorkflowParameters):
6464
return self.run(parameters)
6565

6666
if __name__ == "__main__":
67-
from src.core.integrations import YourIntegration, YourIntegrationConfiguration
67+
from src.core.modules.common.integrations import YourIntegration, YourIntegrationConfiguration
6868

6969
# Initialize integration
7070
integration = YourIntegration(YourIntegrationConfiguration(attribute_1=secret.get("YOUR_SECRET_1"), attribute_2=secret.get("YOUR_SECRET_2")))

‎lib/abi/utils/OntologyYaml.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from abi import logger
88
import json
99
from pathlib import Path
10-
from src.core.ontologies.mapping import MAPPING_URL_LABEL
10+
from src.core.modules.common.ontologies.mapping import MAPPING_URL_LABEL
1111
from rdflib import Graph
1212

1313

‎pyproject.toml

+1-57
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ wordcloud = "^1.9.4"
4343
imapclient = "^3.0.1"
4444
pynacl = "^1.5.0"
4545
python-pptx = "^1.0.2"
46-
#siteanalyzer = { path = "./src/core/integrations/siteanalyzer/target/wheels/siteanalyzer-0.1.0-py3-none-any.whl", develop = true }
46+
#siteanalyzer = { path = "./src/core/modules/common/integrations/siteanalyzer/target/wheels/siteanalyzer-0.1.0-py3-none-any.whl", develop = true }
4747

4848
[tool.poetry.group.terminal-agent.dependencies]
4949
langchain-anthropic = "^0.3.0"
@@ -55,62 +55,6 @@ pytest = "^8.3.4"
5555
ipykernel = "^6.29.5"
5656

5757
[tool.poetry.scripts]
58-
# Foundation agents
59-
chat-supervisor-agent = "src.core.apps.terminal_agent.main:run_supervisor_agent"
60-
chat-support-agent = "src.core.apps.terminal_agent.main:run_support_agent"
61-
62-
# Domain agents
63-
chat-content-agent = "src.core.apps.terminal_agent.main:run_content_agent"
64-
chat-finance-agent = "src.core.apps.terminal_agent.main:run_finance_agent"
65-
chat-growth-agent = "src.core.apps.terminal_agent.main:run_growth_agent"
66-
chat-opendata-agent = "src.core.apps.terminal_agent.main:run_opendata_agent"
67-
chat-operations-agent = "src.core.apps.terminal_agent.main:run_operations_agent"
68-
chat-sales-agent = "src.core.apps.terminal_agent.main:run_sales_agent"
69-
70-
# Integration agents
71-
chat-airtable-agent = "src.core.apps.terminal_agent.main:run_airtable_agent"
72-
chat-algolia-agent = "src.core.apps.terminal_agent.main:run_algolia_agent"
73-
chat-aws-s3-agent = "src.core.apps.terminal_agent.main:run_aws_s3_agent"
74-
chat-clockify-agent = "src.core.apps.terminal_agent.main:run_clockify_agent"
75-
chat-discord-agent = "src.core.apps.terminal_agent.main:run_discord_agent"
76-
chat-github-agent = "src.core.apps.terminal_agent.main:run_github_agent"
77-
chat-gladia-agent = "src.core.apps.terminal_agent.main:run_gladia_agent"
78-
chat-gmail-agent = "src.core.apps.terminal_agent.main:run_gmail_agent"
79-
chat-google-analytics-agent = "src.core.apps.terminal_agent.main:run_google_analytics_agent"
80-
chat-google-calendar-agent = "src.core.apps.terminal_agent.main:run_google_calendar_agent"
81-
chat-google-drive-agent = "src.core.apps.terminal_agent.main:run_google_drive_agent"
82-
chat-google-sheets-agent = "src.core.apps.terminal_agent.main:run_google_sheets_agent"
83-
chat-harvest-agent = "src.core.apps.terminal_agent.main:run_harvest_agent"
84-
chat-hubspot-agent = "src.core.apps.terminal_agent.main:run_hubspot_agent"
85-
chat-linkedin-agent = "src.core.apps.terminal_agent.main:run_linkedin_agent"
86-
chat-naas-agent = "src.core.apps.terminal_agent.main:run_naas_agent"
87-
chat-news-api-agent = "src.core.apps.terminal_agent.main:run_news_api_agent"
88-
chat-notion-agent = "src.core.apps.terminal_agent.main:run_notion_agent"
89-
chat-onedrive-agent = "src.core.apps.terminal_agent.main:run_onedrive_agent"
90-
chat-pennylane-agent = "src.core.apps.terminal_agent.main:run_pennylane_agent"
91-
chat-pipedrive-agent = "src.core.apps.terminal_agent.main:run_pipedrive_agent"
92-
chat-postgres-agent = "src.core.apps.terminal_agent.main:run_postgres_agent"
93-
chat-qonto-agent = "src.core.apps.terminal_agent.main:run_qonto_agent"
94-
chat-serper-agent = "src.core.apps.terminal_agent.main:run_serper_agent"
95-
chat-slack-agent = "src.core.apps.terminal_agent.main:run_slack_agent"
96-
chat-stripe-agent = "src.core.apps.terminal_agent.main:run_stripe_agent"
97-
chat-supabase-agent = "src.core.apps.terminal_agent.main:run_supabase_agent"
98-
chat-yahoo-finance-agent = "src.core.apps.terminal_agent.main:run_yahoo_finance_agent"
99-
chat-youtube-agent = "src.core.apps.terminal_agent.main:run_youtube_agent"
100-
chat-zerobounce-agent = "src.core.apps.terminal_agent.main:run_zerobounce_agent"
101-
chat-sendgrid-agent = "src.core.apps.terminal_agent.main:run_sendgrid_agent"
102-
chat-mailchimp-agent = "src.core.apps.terminal_agent.main:run_mailchimp_agent"
103-
chat-mercury-agent = "src.core.apps.terminal_agent.main:run_mercury_agent"
104-
chat-agicap-agent = "src.core.apps.terminal_agent.main:run_agicap_agent"
105-
chat-brevo-agent = "src.core.apps.terminal_agent.main:run_brevo_agent"
106-
chat-whatsapp-agent = "src.core.apps.terminal_agent.main:run_whatsapp_agent"
107-
chat-instagram-agent = "src.core.apps.terminal_agent.main:run_instagram_agent"
108-
chat-glassdoor-agent = "src.core.apps.terminal_agent.main:run_glassdoor_agent"
109-
chat-powerpoint-agent = "src.core.apps.terminal_agent.main:run_powerpoint_agent"
110-
111-
# Analytics agents
112-
chat-plotly-agent = "src.core.apps.terminal_agent.main:run_plotly_agent"
113-
chat-matplotlib-agent = "src.core.apps.terminal_agent.main:run_matplotlib_agent"
11458
abi = "src.cli:main"
11559
api = "src.api:api"
11660

0 commit comments

Comments
 (0)