Merge branch 'staging' into multistage-poetry-build

Author: Alan Christie

.dockerignore

@@ -2,7 +2,6 @@
 .git/
 venv/
 docs/
-doc_templates/
 design_docs/
 logs/
 media/

.github/workflows/build-dev.yaml

@@ -73,7 +73,7 @@ jobs:
       with:
         dockerfile: Dockerfile
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.11'
     - name: Run pre-commit (all files)

.github/workflows/build-production.yaml

@@ -120,7 +120,7 @@ jobs:
       with:
         dockerfile: Dockerfile
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.11'
     - name: Run pre-commit (all files)

.github/workflows/build-staging.yaml

@@ -142,7 +142,7 @@ jobs:
       with:
         dockerfile: Dockerfile
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.11'
     - name: Run pre-commit (all files)

.pylintrc

@@ -12,3 +12,4 @@ disable = C, F, R,
           fixme,
           import-error,
           imported-auth-user,
+	      new-db-field-with-default,

Dockerfile

@@ -47,7 +47,6 @@ WORKDIR /code
 COPY nginx.conf /etc/nginx/nginx.conf
 COPY django_nginx.conf /etc/nginx/sites-available/default.conf
 COPY proxy_params /etc/nginx/frag_proxy_params
-
 RUN ln -s /etc/nginx/sites-available/default.conf /etc/nginx/sites-enabled
 
 COPY . ./

api/remote_ispyb_connector.py

@@ -1,3 +1,4 @@
+import logging
 import threading
 import time
 import traceback
@@ -11,6 +12,8 @@
     ISPyBRetrieveFailed,
 )
 
+logger: logging.Logger = logging.getLogger(__name__)
+
 
 class SSHConnector(Connector):
     def __init__(
@@ -33,6 +36,7 @@ def __init__(
 
         self.conn_inactivity = conn_inactivity
         self.lock = threading.Lock()
+        self.conn = None
         self.server = None
         self.last_activity_ts = None
 
@@ -48,6 +52,12 @@ def __init__(
                 'db_name': db,
             }
             self.remote_connect(**creds)
+            logger.debug(
+                "Started host=%s username=%s local_bind_port=%s",
+                ssh_host,
+                ssh_user,
+                self.server.local_bind_port,
+            )
 
         else:
             self.connect(
@@ -58,12 +68,14 @@ def __init__(
                 port=port,
                 conn_inactivity=conn_inactivity,
             )
+            logger.debug("Started host=%s user=%s port=%s", host, user, port)
 
     def remote_connect(
         self, ssh_host, ssh_user, ssh_pass, db_host, db_port, db_user, db_pass, db_name
     ):
         sshtunnel.SSH_TIMEOUT = 10.0
         sshtunnel.TUNNEL_TIMEOUT = 10.0
+        sshtunnel.DEFAULT_LOGLEVEL = logging.CRITICAL
         self.conn_inactivity = int(self.conn_inactivity)
 
         self.server = sshtunnel.SSHTunnelForwarder(
@@ -123,3 +135,11 @@ def call_sp_retrieve(self, procname, args):
         if result == []:
             raise ISPyBNoResultException
         return result
+
+    def stop(self):
+        if self.server is not None:
+            self.server.stop()
+        self.server = None
+        self.conn = None
+        self.last_activity_ts = None
+        logger.debug("Server stopped")

api/security.py

@@ -6,6 +6,7 @@
 from typing import Any, Dict, Optional, Union
 from wsgiref.util import FileWrapper
 
+from django.conf import settings
 from django.db.models import Q
 from django.http import Http404, HttpResponse
 from ispyb.connector.mysqlsp.main import ISPyBMySQLSPConnector as Connector
@@ -91,6 +92,7 @@ def get_conn() -> Optional[Connector]:
     #          Assume the credentials are invalid if there is no host.
     #          If a host is not defined other properties are useless.
     if not credentials["host"]:
+        logger.debug("No ISPyB host - cannot return a connector")
         return None
 
     conn: Optional[Connector] = None
@@ -114,19 +116,28 @@ def get_configured_connector() -> Optional[Union[Connector, SSHConnector]]:
     return None
 
 
+def ping_configured_connector() -> bool:
+    """Pings the connector. If a connection can be obtained it is immediately closed.
+    The ping simply provides a way to check the credentials are valid and
+    a connection can be made.
+    """
+    conn: Optional[Union[Connector, SSHConnector]] = None
+    if connector == 'ispyb':
+        conn = get_conn()
+    elif connector == 'ssh_ispyb':
+        conn = get_remote_conn()
+        if conn is not None:
+            conn.stop()
+    return conn is not None
+
+
 class ISpyBSafeQuerySet(viewsets.ReadOnlyModelViewSet):
     def get_queryset(self):
         """
         Optionally restricts the returned purchases to a given proposals
         """
         # The list of proposals this user can have
         proposal_list = self.get_proposals_for_user(self.request.user)
-        # Add in the ones everyone has access to
-        # (unless we already have it)
-        for open_proposal in self.get_open_proposals():
-            if open_proposal not in proposal_list:
-                proposal_list.append(open_proposal)
-
         logger.debug(
             'is_authenticated=%s, proposal_list=%s',
             self.request.user.is_authenticated,
@@ -141,12 +152,10 @@ def get_queryset(self):
     def get_open_proposals(self):
         """
         Returns the list of proposals anybody can access.
+        They are defined via an environment variable
+        and are made available as a list of strings (Project titles)
         """
-        if os.environ.get("TEST_SECURITY_FLAG", False):
-            return ["lb00000"]
-        else:
-            # All of well-known (built-in) public Projects (Proposals/Visits)
-            return ["lb27156"]
+        return settings.PUBLIC_TAS_LIST
 
     def get_proposals_for_user_from_django(self, user):
         # Get the list of proposals for the user
@@ -278,20 +287,27 @@ def get_proposals_for_user(self, user):
         """Returns a list of proposals (public and private) that the user has access to."""
         assert user
 
+        proposals = []
         ispyb_user = os.environ.get("ISPYB_USER")
         logger.debug("ispyb_user=%s", ispyb_user)
         if ispyb_user:
             if user.is_authenticated:
                 logger.info("Getting proposals from ISPyB...")
-                return self.get_proposals_for_user_from_ispyb(user)
+                proposals = self.get_proposals_for_user_from_ispyb(user)
             else:
-                logger.info(
-                    "No proposals (user %s is not authenticated)", user.username
-                )
-                return []
+                username = user.username or "UNKNOWN"
+                logger.info("No proposals (user '%s' is not authenticated)", username)
         else:
             logger.info("Getting proposals from Django...")
-            return self.get_proposals_for_user_from_django(user)
+            proposals = self.get_proposals_for_user_from_django(user)
+
+        # Now add in Target Access Strings that everyone has access to
+        # (unless we already have them)
+        for open_proposal in self.get_open_proposals():
+            if open_proposal not in proposals:
+                proposals.append(open_proposal)
+
+        return proposals
 
     def get_q_filter(self, proposal_list):
         """Returns a Q expression representing a (potentially complex) table filter."""

api/utils.py

@@ -399,3 +399,7 @@ def pretty_request(request, *, tag='', print_body=False):
         f'{body}\n'
         '- REQUEST END'
     )
+
+
+def deployment_mode_is_production():
+    return settings.DEPLOYMENT_MODE == "PRODUCTION"

docker-compose.yml

@@ -86,6 +86,7 @@ services:
     - .:/code/
     environment:
       AUTHENTICATE_UPLOAD: ${AUTHENTICATE_UPLOAD:-True}
+      DEPLOYMENT_MODE: 'development'
       POSTGRESQL_USER: postgres
       # Celery tasks need to run synchronously
       CELERY_TASK_ALWAYS_EAGER: 'True'
@@ -99,6 +100,9 @@ services:
       OIDC_AS_CLIENT_ID: ${OIDC_AS_CLIENT_ID:-account-server-api}
       OIDC_DM_CLIENT_ID: ${OIDC_DM_CLIENT_ID:-data-manager-api}
       OIDC_RENEW_ID_TOKEN_EXPIRY_MINUTES: '210'
+      # Public target access strings?
+      # A comma-separated list of Project titles.
+      PUBLIC_TAS: lb18145-1
       # Squonk configuration
       SQUONK2_VERIFY_CERTIFICATES: 'No'
       SQUONK2_UNIT_BILLING_DAY: 3

fragalysis/settings.py

@@ -237,6 +237,12 @@
 OIDC_RENEW_ID_TOKEN_EXPIRY_SECONDS = int(TOKEN_EXPIRY_MINUTES) * 60
 # Keycloak mozilla_django_oidc - Settings - End
 
+# The deployment mode.
+# Controls the behaviour of the application (it's strictness to errors etc).
+# Typically one of "DEVELOPMENT" or "PRODUCTION".
+# see api.utils for the 'deployment_mode_is_production()' function.
+DEPLOYMENT_MODE = os.environ.get("DEPLOYMENT_MODE", "production").upper()
+
 ROOT_URLCONF = "fragalysis.urls"
 
 STATIC_ROOT = os.path.join(PROJECT_ROOT, "static")
@@ -389,6 +395,10 @@
     "TAS_REGEX_ERROR_MSG",
     "Must begin 'lb' followed by 5 digits, optionally followed by a hyphen and a number.",
 )
+# Are any public target access strings defined?
+# If so they'll be in the PUBLIC_TAS variable as a comma separated list.
+PUBLIC_TAS = os.environ.get("PUBLIC_TAS", "")
+PUBLIC_TAS_LIST = PUBLIC_TAS.split(",") if PUBLIC_TAS else []
 
 COMPUTED_SET_MEDIA_DIRECTORY = "computed_set_data"
 TARGET_LOADER_MEDIA_DIRECTORY = "target_loader_data"
@@ -447,9 +457,9 @@
             'api.security': {'level': 'INFO'},
             'asyncio': {'level': 'WARNING'},
             'celery': {'level': 'INFO'},
-            'django': {'level': 'INFO'},
-            'mozilla_django_oidc': {'level': 'INFO'},
-            'urllib3': {'level': 'INFO'},
+            'django': {'level': 'WARNING'},
+            'mozilla_django_oidc': {'level': 'WARNING'},
+            'urllib3': {'level': 'WARNING'},
             'paramiko': {'level': 'WARNING'},
         },
         'root': {

nginx.conf

@@ -27,7 +27,6 @@ http {
         sendfile on;
         tcp_nopush on;
         tcp_nodelay on;
-        keepalive_timeout 650;
         types_hash_max_size 2048;
         # server_tokens off;
 
@@ -51,7 +50,8 @@ http {
         access_log /var/log/nginx/access.log;
         error_log /var/log/nginx/error.log;
 
-        # Tuneout settings
+        # Timeeout settings
+        keepalive_timeout           1000s;
         proxy_connect_timeout       1000s;
         proxy_send_timeout          1000s;
         proxy_read_timeout          1000s;