Docker Image Change to Amazon Linux (#1207)

* Update to use new Amazon Linux base image and use the same structure as our other python services. * Utilizing "gen3" user instead of "root" for more secure containers * Moving to Poetry to manage our virtual environments * Multi-stage Docker builds for smaller images * Move to Gunicorn --------- Co-authored-by: Jawad Qureshi <qureshi@uchicago.edu> Co-authored-by: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Co-authored-by: Alexander VanTol <avantol@uchicago.edu> Co-authored-by: Alexander VanTol <Avantol13@users.noreply.github.com> Co-authored-by: Sai Shanmukha <nss10@outlook.com> Co-authored-by: J. Q. <55899496+jawadqur@users.noreply.github.com>
uc-cdis · Feb 19, 2025 · b2fa0ae · b2fa0ae
1 parent 7dcce30
commit b2fa0ae
Show file tree

Hide file tree

Showing 16 changed files with 425 additions and 398 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -10,19 +10,22 @@ jobs:
   Security:
     name: Security Pipeline
     uses: uc-cdis/.github/.github/workflows/securitypipeline.yaml@master
-    secrets: inherit
+    secrets: inherit  # pragma: allowlist secret
 
   UnitTest:
     name: Python Unit Test with Postgres
     uses: uc-cdis/.github/.github/workflows/python_unit_test.yaml@master
     with:
-       python-version: '3.9'
-       test-script: 'tests/ci_commands_script.sh'
-       run-coveralls: true
-  ci:
+      python-version: '3.9'
+      test-script: 'tests/ci_commands_script.sh'
+      run-coveralls: true
+
+  BuildImageAndPush:
     name: Build Image and Push
-    # TODO Uncomment after PXP-9212
-    # needs: Security
+    needs: Security
+    with:
+      BUILD_PLATFORMS: "linux/amd64"
+    # https://github.com/uc-cdis/.github/blob/master/.github/workflows/image_build_push.yaml
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     secrets:
       ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}

diff --git a/.secrets.baseline b/.secrets.baseline
@@ -115,13 +115,13 @@
     }
   ],
   "results": {
-    ".github/workflows/ci.yaml": [
+    ".github/workflows/buildpipeline.yaml": [
       {
         "type": "Secret Keyword",
-        "filename": ".github/workflows/ci.yaml",
+        "filename": ".github/workflows/buildpipeline.yaml",
         "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
         "is_verified": false,
-        "line_number": 13
+        "line_number": 17
       }
     ],
     "deployment/scripts/postgresql/postgresql_init.sql": [
@@ -210,15 +210,6 @@
         "line_number": 137
       }
     ],
-    "fence/resources/storage/storageclient/cleversafe.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "fence/resources/storage/storageclient/cleversafe.py",
-        "hashed_secret": "7cb6efb98ba5972a9b5090dc2e517fe14d12cb04",
-        "is_verified": false,
-        "line_number": 274
-      }
-    ],
     "fence/utils.py": [
       {
         "type": "Secret Keyword",
@@ -268,14 +259,14 @@
         "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1570
+        "line_number": 1556
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1594
+        "line_number": 1579
       }
     ],
     "tests/credentials/google/test_credentials.py": [
@@ -394,24 +385,6 @@
         "line_number": 300
       }
     ],
-    "tests/storageclient/storage_client_mock.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/storageclient/storage_client_mock.py",
-        "hashed_secret": "37bbea9557f9efd1eeadb25dda9ab6514f08fde9",
-        "is_verified": false,
-        "line_number": 158
-      }
-    ],
-    "tests/storageclient/test_cleversafe_api_client.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/storageclient/test_cleversafe_api_client.py",
-        "hashed_secret": "f683c485d521c2e45830146dd570111770baea29",
-        "is_verified": false,
-        "line_number": 130
-      }
-    ],
     "tests/test-fence-config.yaml": [
       {
         "type": "Basic Auth Credentials",
@@ -422,5 +395,5 @@
       }
     ]
   },
-  "generated_at": "2024-11-04T09:20:13Z"
+  "generated_at": "2025-01-03T16:46:44Z"
 }
diff --git a/Dockerfile b/Dockerfile
@@ -1,56 +1,63 @@
-# To run: docker run --rm -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence
-# To check running container do: docker exec -it fence /bin/bash
+# To build: docker build -t fence:latest .
+# To run interactive:
+#   docker run -v ~/.gen3/fence/fence-config.yaml:/var/www/fence/fence-config.yaml -v ./keys/:/fence/keys/ fence:latest
+# To check running container do: docker exec -it CONTAINER bash
 
-FROM quay.io/cdis/python:python3.9-buster-2.0.0
+ARG AZLINUX_BASE_VERSION=master
+
+# ------ Base stage ------
+FROM quay.io/cdis/python-nginx-al:${AZLINUX_BASE_VERSION} AS base
+
+# Comment this in, and comment out the line above, if quay is down
+# FROM 707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/python-nginx-al:${AZLINUX_BASE_VERSION} as base
 
 ENV appname=fence
 
-RUN pip install --upgrade pip
-RUN pip install --upgrade poetry
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl bash git \
-    && apt-get -y install vim \
-    libmcrypt4 mcrypt \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/
-
-RUN mkdir -p /var/www/$appname \
-    && mkdir -p /var/www/.cache/Python-Eggs/ \
-    && mkdir /run/nginx/ \
-    && ln -sf /dev/stdout /var/log/nginx/access.log \
-    && ln -sf /dev/stderr /var/log/nginx/error.log \
-    && chown nginx -R /var/www/.cache/Python-Eggs/ \
-    && chown nginx /var/www/$appname
-
-# aws cli v2 - needed for storing files in s3 during usersync k8s job
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
-    && unzip awscliv2.zip \
-    && ./aws/install \
-    && /bin/rm -rf awscliv2.zip ./aws
-
-WORKDIR /$appname
-
-# copy ONLY poetry artifact, install the dependencies but not fence
-# this will make sure than the dependencies is cached
-COPY poetry.lock pyproject.toml /$appname/
-RUN poetry config virtualenvs.create false \
-    && poetry install -vv --no-root --without dev --no-interaction \
-    && poetry show -v
-
-# copy source code ONLY after installing dependencies
-COPY . /$appname
-COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
-COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
-COPY clear_prometheus_multiproc /$appname/clear_prometheus_multiproc
-
-# install fence
-RUN poetry config virtualenvs.create false \
-    && poetry install -vv --without dev --no-interaction \
-    && poetry show -v
-
-RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py
-
-WORKDIR /var/www/$appname
-
-CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"]
+WORKDIR /${appname}
+
+RUN chown -R gen3:gen3 /${appname}
+
+# ------ Builder stage ------
+FROM base AS builder
+
+# Install just the deps without the code as it's own step to avoid redoing this on code changes
+COPY poetry.lock pyproject.toml /${appname}/
+RUN poetry lock -vv --no-update \
+    && poetry install -vv --only main --no-interaction
+
+# Move app files into working directory
+COPY --chown=gen3:gen3 . /$appname
+COPY --chown=gen3:gen3 ./deployment/wsgi/wsgi.py /$appname/wsgi.py
+
+# Do the install again incase the app itself needs install
+RUN poetry lock -vv --no-update \
+    && poetry install -vv --only main --no-interaction
+
+# Setup version info
+RUN git config --global --add safe.directory /${appname} && COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" > /$appname/version_data.py \
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >> /$appname/version_data.py
+
+
+
+# ------ Final stage ------
+FROM base
+
+ENV PATH="/${appname}/.venv/bin:$PATH"
+
+# Install ccrypt to decrypt dbgap telmetry files
+RUN echo "Upgrading dnf"; \
+    dnf upgrade -y; \
+    echo "Installing Packages"; \
+    dnf install -y \
+        libxcrypt-compat-4.4.33 \
+        libpq-15.0 \
+        gcc \
+        tar xz; \
+    echo "Installing RPM"; \
+    rpm -i https://ccrypt.sourceforge.net/download/1.11/ccrypt-1.11-1.src.rpm && \
+    cd /root/rpmbuild/SOURCES/ && \
+    tar -zxf ccrypt-1.11.tar.gz && cd ccrypt-1.11 && ./configure --disable-libcrypt && make install && make check;
+
+COPY --chown=gen3:gen3 --from=builder /$appname /$appname
+
+CMD ["/bin/bash", "-c", "/fence/dockerrun.bash"]
diff --git a/deployment/fence.conf b/deployment/fence.conf
diff --git a/deployment/wsgi/gunicorn.conf.py b/deployment/wsgi/gunicorn.conf.py
@@ -0,0 +1,9 @@
+wsgi_app = "deployment.wsgi.wsgi:application"
+bind = "0.0.0.0:8000"
+workers = 1
+preload_app = True
+user = "root"
+group = "root"
+timeout = 300
+keepalive = 2
+keepalive_timeout = 5
diff --git a/deployment/uwsgi/wsgi.py → deployment/wsgi/wsgi.py b/deployment/uwsgi/wsgi.py → deployment/wsgi/wsgi.py
diff --git a/dockerrun.bash b/dockerrun.bash
@@ -1,10 +1,5 @@
 #!/bin/bash
 
-#
-# Update certificate authority index -
-# environment may have mounted more authorities
-#
-update-ca-certificates
 #
 # Kubernetes may mount jwt-keys as a tar ball
 #
@@ -18,3 +13,6 @@ if [ -f /fence/jwt-keys.tar ]; then
     fi
   )
 fi
+
+nginx
+poetry run gunicorn -c "/fence/deployment/wsgi/gunicorn.conf.py"
diff --git a/dockerrunshib.bash b/dockerrunshib.bash
diff --git a/fence/config.py b/fence/config.py
@@ -41,6 +41,7 @@ def post_process(self):
             "WHITE_LISTED_GOOGLE_PARENT_ORGS",
             "CLIENT_CREDENTIALS_ON_DOWNLOAD_ENABLED",
             "DATA_UPLOAD_BUCKET",
+            "DEFAULT_BACKOFF_SETTINGS_MAX_TRIES",
         ]
         for default in defaults:
             self.force_default_if_none(default, default_cfg=default_config)

diff --git a/fence/jwt/keys.py b/fence/jwt/keys.py
@@ -148,13 +148,26 @@ def from_directory(cls, keys_dir, naming_function=None):
         prv_filepath = os.path.join(keys_dir, "jwt_private_key.pem")
 
         if not os.path.isfile(pub_filepath):
-            raise EnvironmentError(
-                "missing public key file; expected file to exist: " + pub_filepath
-            )
+            # Generate public key from private key
+            with open(prv_filepath, "r") as f:
+                private_key_file = f.read()
+                private_key = serialization.load_pem_private_key(
+                    bytes(private_key_file, "utf-8"),
+                    password=None,
+                    backend=default_backend(),
+                )
+                public_key = private_key.public_key()
+                public_key = public_key.public_bytes(
+                    encoding=serialization.Encoding.PEM,
+                    format=serialization.PublicFormat.SubjectPublicKeyInfo,
+                )
+                public_key = public_key.decode("utf-8")
+                with open(pub_filepath, "w") as f:
+                    f.write(public_key)
 
         if not os.path.isfile(prv_filepath):
             raise EnvironmentError(
-                "missing public key file; expected file to exist: " + prv_filepath
+                "missing private key file; expected file to exist: " + prv_filepath
             )
 
         with open(pub_filepath, "r") as f:

diff --git a/fence/jwt/token.py b/fence/jwt/token.py
@@ -179,7 +179,7 @@ def generate_signed_session_token(kid, private_key, expires_in, context=None):
     claims = {
         "pur": "session",
         "aud": ["fence", issuer],
-        "sub": context.get("user_id", ""),
+        "sub": str(context.get("user_id", "")),
         "iss": issuer,
         "iat": iat,
         "exp": exp,

diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py
@@ -100,30 +100,14 @@ def _read_file(filepath, encrypted=True, key=None, logger=None):
         Generator[file-like class]: file like object for the file
     """
     if encrypted:
-        has_crypt = sp.call(["which", "mcrypt"])
-        if has_crypt != 0:
-            if logger:
-                logger.error("Need to install mcrypt to decrypt files from dbgap")
-            # TODO (rudyardrichter, 2019-01-08): raise error and move exit out to script
-            exit(1)
         p = sp.Popen(
             [
-                "mcrypt",
-                "-a",
-                "enigma",
-                "-o",
-                "scrypt",
-                "-m",
-                "stream",
-                "--bare",
-                "--key",
+                "ccdecrypt",
+                "-u",
+                "-K",
                 key,
-                "--force",
-            ],
-            stdin=open(filepath, "r"),
-            stdout=sp.PIPE,
-            stderr=open(os.devnull, "w"),
-            universal_newlines=True,
+                filepath,
+            ]
         )
         try:
             yield StringIO(p.communicate()[0])
@@ -402,7 +386,7 @@ def _get_from_sftp_with_proxy(self, server, path):
         """
         proxy = None
         if server.get("proxy", "") != "":
-            command = "ssh -i ~/.ssh/id_rsa {user}@{proxy} nc {host} {port}".format(
+            command = "ssh -oHostKeyAlgorithms=+ssh-rsa -i ~/.ssh/id_rsa {user}@{proxy} nc {host} {port}".format(
                 user=server.get("proxy_user", ""),
                 proxy=server.get("proxy", ""),
                 host=server.get("host", ""),