Switch to openapi and postgres

Author: killua-eu

README.md

@@ -1,3 +1,51 @@
 # glued-if
 
-Glued Integration Framework
+IF is Glued's integration framework, message queue and scheduler.
+
+**Implementation Overview:**
+
+- `if microservice` is a set of methods implemented as a separate git project `glued-if-service_name` to interact with a foreign service
+- `deployment` is a configuration and authorization context of an `if microservice` defined by a row/object of the `if__deployments` table
+- `action` is a `deployment:microservice.method` map stored as a row in the `if__actions` table. actions are triggered by calling an API endpoint (webhook) or by the IF scheduler.
+- `action state` makes a stateful history using `if__actions_states` 
+- `scheduler`is implemented as a set of cron-like rules with some extra sugarcoating (i.e. ttl) stored in the `if__scheduler` table and an associated producer that will periodically fill up the `message queue` with work and runner daemons that will subscribe to this queue and launch workers (perform the webhook api calls)
+- `message queue` is implemented as the `if__mq_queue` table configuring message queues clients can subscribe to.
+- `messages` are implemented as the `if__mq_messages` table which maps a queue to a message payload and message headers (i.e. ttl, requesting, replies, etc.)
+- `notifications` are implemented using the pg_notify capabilities and can be further extended with rabbitmq. 
+- `logging` is by default implemented using the monolog library.
+
+## Integrations
+
+IF orchestrates `if microservices` which facilitate integration with anything thats out there.
+
+- Each `if microservice` implements working with a remote/external `service`.
+- Each `if microservice` has associated `deployments` (a set of attributes describing usage of an `if microservice`), i.e:
+  - deployment metadata (i.e. name, description, etc.)
+  - deployment connection (i.e. remote host, auth tokens, rate limits, etc.)
+  - deployment RBAC rules* (i.e. who can use the if microservice)
+- Methods implemented by an `if microservice` (i.e. CRUD operations against an external service) are associated to deployments as `actions` (actions are if microservice methods runnable in the context of a deployment configuration). actions
+  - can run on-demand (i.e. provide a caching data transforming interface to external services such as glued-if-ares_gov_cz)
+  - can run according to a schedule (by the scheduler) 
+  - can interact with each other (via the message queue)
+
+*) RBAC is provided by glued-core's authorization proxy
+
+## Scheduler
+
+tbd
+
+## Message queue
+
+The message queue is loosely inspired by RabbitMQ to enable an easy transition between the builtin
+PostgreSQL based queue and Rabbit. The indented MQ usage is
+- to distribute scheduled tasks to workers,
+- to facilitate communication between the loosely coupled microservices
+- to ensure internal notifications to users are send and individually delivered (with or without user confirmation)
+
+IFMQ, when acting as a RabbitMQ-like message brooker, implements the concepts of
+- `producers` / code responsible for generating messages - php implementation is part of glued-lib
+- `queues` / 'message destinations' implemented via the `if__queues` table
+- `consumers` / clients subscribed to queues
+- `exchanges` / code responsible for delivering messages to queues - direct (unicast), fanout (multicast), header (rule based multicast)
+
+Messges can be 

composer.json

@@ -1,5 +1,5 @@
 {
-  "name": "vaizard/glued-integrations",
+  "name": "vaizard/glued-if",
   "description": "Integrations manager.",
   "license": "MIT",
   "authors": [
@@ -9,7 +9,7 @@
     }
   ],
   "require": {
-    "php": "^8.2",
+    "php": "^8.3",
     "ext-apcu": "*",
     "ext-bcmath": "*",
     "ext-curl": "*",
@@ -22,38 +22,26 @@
     "ext-readline": "*",
     "ext-soap": "*",
     "ext-xml": "*",
-    "grasmash/yaml-expander": "^3",
-    "league/flysystem": "^3",
-    "monolog/monolog": "^3",
+    "ext-pdo": "*",
+    "ext-pdo_pgsql": "*",
     "nyholm/psr7": "^1",
     "nyholm/psr7-server": "*",
-    "opis/json-schema": "^2",
     "php-di/php-di": "^7",
     "psr/http-message": "^1",
-    "phpfastcache/phpfastcache": "^9",
     "ramsey/uuid": "^4",
     "sabre/event": "^6",
-    "selective/transformer": "^1",
     "slim/http": "^1",
     "slim/slim": "^4",
-    "symfony/yaml": "^6",
-    "thingengineer/mysqli-database-class": "dev-master#42116651a88b57b03fd967bc57dd38f5094565d9",
     "vaizard/glued-lib": "dev-main",
-    "zeuxisoo/slim-whoops": "^0.7"
+    "zeuxisoo/slim-whoops": "^0.7",
+    "league/openapi-psr7-validator": "^0.22.0"
   },
   "require-dev": {
     "ergebnis/composer-normalize": "^2"
   },
   "suggest": {
     "ext-xdebug": "For dev only"
   },
-  "repositories": [
-    {
-      "type": "vcs",
-      "url": "https://github.com/ThingEngineer/PHP-MySQLi-Database-Class",
-      "no-api": true
-    }
-  ],
   "minimum-stability": "beta",
   "prefer-stable": true,
   "autoload": {
@@ -83,8 +71,8 @@
     ],
     "configure": [
       "vendor/vaizard/glued-lib/src/Scripts/initpaths.sh",
-      "Glued\\Lib\\ComposerHooks::configTool",
-      "vendor/vaizard/glued-lib/src/Scripts/cacheroutes.sh"
+      "vendor/vaizard/glued-lib/src/Scripts/rebuild-datacache.sh",
+      "Glued\\Lib\\ComposerHooks::configTool"
     ],
     "migrate": [
       "vendor/vaizard/glued-lib/src/Scripts/migrate.sh"

glued/Config/Migrations/20231009002534_actions.sql renamed to glued/Config/Mysql/20231009002534_actions.sql

@@ -0,0 +1,29 @@
+-- migrate:up
+
+CREATE TABLE "glued"."if__deployments" (
+    uuid uuid generated always as (((doc ->> 'uuid'::text))::uuid) stored not null,
+    doc jsonb not null,
+    nonce bytea generated always as ( decode( md5((doc - 'uuid')::text), 'hex')) stored,
+    created_at timestamp with time zone default CURRENT_TIMESTAMP,
+    updated_at timestamp with time zone default CURRENT_TIMESTAMP,
+    service text generated always as (doc->>'service') stored,
+    name text generated always as (doc->>'name') stored,
+    description text generated always as (doc->>'description') stored,
+    PRIMARY KEY ("uuid")
+);
+
+CREATE INDEX idx_service ON glued.if__deployments ("service");
+CREATE INDEX idx_name ON glued.if__deployments ("name");
+ALTER TABLE "glued"."if__deployments" ADD CONSTRAINT unique_nonce UNIQUE (nonce);
+
+COMMENT ON TABLE "glued"."if__deployments" IS 'Integration framework service deployments.';
+COMMENT ON COLUMN "glued"."if__deployments"."uuid" IS 'IF service deployment uuid, autogenerated on SQL insert if not provided.';
+COMMENT ON COLUMN "glued"."if__deployments"."doc" IS 'Connection configuration, i.e. host, port, token, password, etc. (encrypted)';
+COMMENT ON COLUMN "glued"."if__deployments"."nonce" IS 'MD5 hash of (doc - uuid) to track changes to the document or act as a unique index';
+COMMENT ON COLUMN "glued"."if__deployments"."service" IS 'Service type';
+COMMENT ON COLUMN "glued"."if__deployments"."name" IS 'Deployment name';
+COMMENT ON COLUMN "glued"."if__deployments"."description" IS 'Deployment description';
+
+-- migrate:down
+
+DROP TABLE "glued"."if__deployments" CASCADE;

glued/Config/Migrations/20231009002540_runs.sql renamed to glued/Config/Mysql/20231009002540_runs.sql

@@ -0,0 +1,31 @@
+-- migrate:up
+
+CREATE TABLE glued.if__actions (
+    uuid uuid generated always as (((doc ->> 'uuid'::text))::uuid) stored not null,
+    doc jsonb not null,
+    nonce bytea generated always as ( decode( md5((doc - 'uuid')::text), 'hex')) stored,
+    created_at timestamp with time zone default CURRENT_TIMESTAMP,
+    updated_at timestamp with time zone default CURRENT_TIMESTAMP,
+    svc_name text generated always as (doc->>'service.name') stored,
+    svc_version text generated always as (doc->>'service.version') stored,
+    svc_version text generated always as (doc->>'service.version') stored,
+
+    us_version varchar(255),
+    us_method varchar(255),
+    deployment_uuid uuid NOT NULL,
+    props jsonb NULL,
+    nonce bytea GENERATED ALWAYS AS ( decode( md5( (deployment_uuid || us_version || us_method || props )::text ), 'hex')) STORED,
+    PRIMARY KEY (uuid),
+    UNIQUE (nonce),
+    FOREIGN KEY (deployment_uuid) REFERENCES glued.if__deployments(uuid)
+);
+
+COMMENT ON TABLE glued.if__actions IS 'IF service actions are service methods available in an deployment configuration/authorization context.';
+COMMENT ON COLUMN glued.if__actions.us_name IS 'IF microservice name (i.e. gov_ares_cz)';
+COMMENT ON COLUMN glued.if__actions.us_version IS 'IF microservice version (i.e. v1)';
+COMMENT ON COLUMN glued.if__actions.us_method IS 'IF microservice method (i.e. search)';
+COMMENT ON COLUMN glued.if__actions.deployment_uuid IS 'IF Deployment reference, a.k.a. the configuration/authorization context to a microservice';
+
+-- migrate:down
+
+DROP TABLE IF EXISTS glued.if__actions;

glued/Config/Migrations/20231009002603_deployments.sql renamed to glued/Config/Mysql/20231009002603_deployments.sql

@@ -0,0 +1,24 @@
+-- migrate:up
+
+CREATE TABLE glued.if__mq_messages (
+    uuid uuid DEFAULT gen_random_uuid() NOT NULL,                       -- message uuid
+    queue_uuid uuid NOT NULL,                                           -- Recipients subscribe to a queue
+    subject text NULL,                                                  -- Message subject, i.e.: `run`
+    payload jsonb NULL,                                                 -- json payload
+    priority integer NOT NULL DEFAULT 1 CHECK (priority IN (1,2,3,4,5)),-- priority in range: 1 .. 5
+    snd_at TIMESTAMP WITHOUT TIME ZONE NULL DEFAULT now(),              -- message created and sent timestamp
+    snd_by uuid NULL,                                                   -- message created and sent author
+    dlv_at TIMESTAMP WITHOUT TIME ZONE NULL,                            -- delivered at timestamp
+    dlv_to uuid NULL,                                                   -- delivered to uuid
+    ack_rq TIMESTAMP WITHOUT TIME ZONE NULL,                            -- acknowledgement is required before time (or isnt required if null)
+    ack_at TIMESTAMP WITHOUT TIME ZONE NULL,                            -- acknowledged at timestamp
+    ack_as TEXT CHECK (ack_as IN ('ack', 'nack', 'reject')),            -- acknowledged as ack (acknowledged / i.e. worker performed an action with an ok status), nack (received but not acknowledged / i.e. worker performed an action, but something failed in the process), reject (rejected / i.e. worker received an invalid payload)
+    rpl_to uuid NULL,                                                   -- replies to be sent to queue uuid
+    nbf_at TIMESTAMP WITHOUT TIME ZONE NULL,                            -- message not before (delayed messages)
+    exp_at TIMESTAMP WITHOUT TIME ZONE NULL,                            -- message expires at (time limit on messages)
+    PRIMARY KEY (uuid, queue_uuid)                                      -- Composite primary key on uuid and queue_uuid
+);
+
+-- migrate:down
+
+DROP TABLE IF EXISTS glued.if__mq_messages;

glued/Config/Migrations/20240122164828_objects.sql renamed to glued/Config/Mysql/20240122164828_objects.sql

@@ -0,0 +1,15 @@
+-- migrate:up
+
+CREATE TABLE glued.if__actions_states (
+    uuid uuid NOT NULL DEFAULT gen_random_uuid(), -- uuid state
+    action_uuid uuid NOT NULL REFERENCES if__actions(uuid) ON DELETE CASCADE, -- message uuid
+    at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    fid varchar(255) NOT NULL,
+    nonce bytea NULL,
+    UNIQUE ("action_uuid", "nonce"),
+    UNIQUE ("action_uuid", "fid")
+);
+
+-- migrate:down
+
+DROP TABLE IF EXISTS glued.if__actions_states;

glued/Config/Pgsql/20240311090752_deployments.sql

@@ -0,0 +1,19 @@
+-- migrate:up
+
+CREATE TABLE glued.if__mq_queues (
+    uuid uuid DEFAULT gen_random_uuid() NOT NULL PRIMARY KEY,
+    name TEXT,
+    description TEXT,
+    created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    ttl integer NULL, -- queue ttl in seconds
+    dlq_uuid UUID, -- Reference to another queue acting as DLQ, leave NULL for a cleanup
+    FOREIGN KEY (dlq_uuid) REFERENCES glued.if__mq_queues(uuid)
+);
+
+COMMENT ON TABLE "glued"."if__mq_queues" IS 'Message queues.';
+COMMENT ON COLUMN "glued"."if__mq_queues"."uuid" IS 'Queue UUID.';
+COMMENT ON COLUMN "glued"."if__mq_queues"."name" IS 'Queue name.';
+
+-- migrate:down
+
+DROP TABLE glued.if__mq_queues;

glued/Config/Pgsql/20240331205651_actions.sql

@@ -0,0 +1,15 @@
+-- migrate:up
+
+CREATE TABLE glued.if__mq_queue_subscribers (
+    queue_uuid uuid NOT NULL,
+    subscriber_uuid uuid NOT NULL,
+    subscriber_kind text NOT NULL,
+    subscriber_note text NULL,
+    PRIMARY KEY (queue_uuid, subscriber_uuid),
+    FOREIGN KEY (queue_uuid) REFERENCES glued.if__mq_queues(uuid)
+);
+
+-- migrate:down
+
+DROP TABLE glued.if__mq_queue_subscribers;
+