PXP-7805 Fetch audit logs from an AWS SQS (#2)

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: git@github.com:Yelp/detect-secrets
-    rev: v1.0.3
+    rev: v0.13.1
     hooks:
     -   id: detect-secrets
         args: ['--baseline', '.secrets.baseline']

.secrets.baseline

@@ -1,5 +1,9 @@
 {
-  "generated_at": "2021-02-25T23:24:34Z",
+  "exclude": {
+    "files": "poetry.lock",
+    "lines": null
+  },
+  "generated_at": "2021-05-19T17:50:55Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -8,8 +12,8 @@
       "name": "ArtifactoryDetector"
     },
     {
-      "name": "Base64HighEntropyString",
-      "limit": 4.5
+      "base64_limit": 4.5,
+      "name": "Base64HighEntropyString"
     },
     {
       "name": "BasicAuthDetector"
@@ -18,8 +22,8 @@
       "name": "CloudantDetector"
     },
     {
-      "name": "HexHighEntropyString",
-      "limit": 3
+      "hex_limit": 3,
+      "name": "HexHighEntropyString"
     },
     {
       "name": "IbmCloudIamDetector"
@@ -56,55 +60,24 @@
   "results": {
     ".github/workflows/ci.yaml": [
       {
-        "type": "Secret Keyword",
-        "filename": ".github/workflows/ci.yaml",
         "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
         "is_verified": false,
-        "line_number": 14
+        "line_number": 14,
+        "type": "Secret Keyword"
       }
     ],
     "tests/test-audit-service-config.yaml": [
       {
-        "type": "Secret Keyword",
-        "filename": "tests/test-audit-service-config.yaml",
         "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
         "is_verified": false,
-        "line_number": 10
+        "line_number": 10,
+        "type": "Secret Keyword"
       }
     ]
   },
-  "version": "1.0.3",
-  "filters_used": [
-    {
-      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_sequential_string"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_templated_secret"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
-    },
-    {
-      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
-      "min_level": 2
-    },
-    {
-      "path": "detect_secrets.filters.regex.should_exclude_file",
-      "pattern": [
-        "poetry.lock"
-      ]
-    }
-  ]
+  "version": "0.13.1",
+  "word_list": {
+    "file": null,
+    "hash": null
+  }
 }

README.md

@@ -23,5 +23,5 @@ The server is built with [FastAPI](https://fastapi.tiangolo.com/) and packaged w
 * [Deploying the Audit Service to a Gen3 Data Commons](docs/how-to/deployment.md)
 * [Architecture](docs/reference/architecture.md)
 * [Query response page size](docs/explanation/query_page_size.md)
-* [Async POST endpoint](docs/explanation/async_post.md)
+* [Creating audit logs](docs/explanation/creating_audit_logs.md)
 * [How to add a new audit log category?](docs/how-to/add_log_category.md)

docs/explanation/creating_audit_logs.md

@@ -0,0 +1,13 @@
+# Creating audit logs
+
+## Async POST endpoint
+
+The audit log creation endpoints is an async endpoint:
+- POSTing audit logs does not impact the performance of the caller.
+- Audit Service failures are not visible to users (for example, we don’t want to return a 500 error to users who are trying to download).
+
+However, it's difficult to monitor errors when using this endpoint.
+
+## Pulling from a queue
+
+The audit service can also handle pulling audit logs from a queue, which allows for easier monitoring. This can be configured by turning on the `PULL_FROM_QUEUE` flag in the configuration file (enabled by default). Right now, only AWS SQS is integrated, but integrations for other types of queues can be added by adding code and extending the values accepted for the `QUEUE_CONFIG.type` field in the configuration file.

docs/how-to/deployment.md

@@ -9,6 +9,8 @@ ENABLE_AUDIT_LOGS:
   login: <true or false>
 ```
 
+The `PUSH_AUDIT_LOGS_CONFIG` field must also be configured. When using `type: aws_sqs`, the SQS URL and region can be copied from the Audit Service configuration. See the [default Fence configuration file](https://github.com/uc-cdis/fence/blob/master/fence/config-default.yaml) for more details.
+
 ## Notes
 
 1. When adding audit log creation in a service for the first time, the `audit-service` deployment file `network-ingress` annotation (see [here](https://github.com/uc-cdis/cloud-automation/blob/27770776d239bc609bbbd23607689cf62de1bc66/kube/services/audit-service/audit-service-deploy.yaml#L6)) must be updated to allow the service to talk to `audit-service`.

docs/openapi.yaml

@@ -73,7 +73,6 @@ components:
       - username
       - sub
       - guid
-      - resource_paths
       - action
       title: CreatePresignedUrlLogInput
       type: object

migrations/versions/fd0510a0a9aa_optional_presigned_url_resource_paths.py

@@ -0,0 +1,30 @@
+"""optional presigned_url resource_paths
+
+Revision ID: fd0510a0a9aa
+Revises: d5b18185c458
+Create Date: 2021-05-25 15:06:06.372742
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "fd0510a0a9aa"
+down_revision = "d5b18185c458"
+branch_labels = None
+depends_on = None
+
+
+table_name = "presigned_url"
+
+
+def upgrade():
+    op.alter_column(table_name, "resource_paths", nullable=True)
+
+
+def downgrade():
+    # replace null values with an empty array
+    op.execute(
+        f"UPDATE {table_name} SET resource_paths=ARRAY[]::VARCHAR[] WHERE resource_paths IS NULL"
+    )
+    op.alter_column(table_name, "resource_paths", nullable=False)