diff --git a/.github/workflows/build-dev.yaml b/.github/workflows/build-dev.yaml index 689f42f5..4bb3aca3 100644 --- a/.github/workflows/build-dev.yaml +++ b/.github/workflows/build-dev.yaml @@ -85,12 +85,11 @@ jobs: with: context: . tags: ${{ steps.vars.outputs.BE_NAMESPACE }}/fragalysis-backend:${{ env.GITHUB_REF_SLUG }} - - name: Test - run: > - docker-compose -f docker-compose.test.yml up - --build - --exit-code-from tests - --abort-on-container-exit + - name: Test (docker compose) + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: ./docker-compose.test.yml + up-flags: --build --exit-code-from tests --abort-on-container-exit env: BE_NAMESPACE: ${{ steps.vars.outputs.BE_NAMESPACE }} BE_IMAGE_TAG: ${{ env.GITHUB_REF_SLUG }} diff --git a/.github/workflows/build-production.yaml b/.github/workflows/build-production.yaml index b49d06df..11cec835 100644 --- a/.github/workflows/build-production.yaml +++ b/.github/workflows/build-production.yaml @@ -134,12 +134,11 @@ jobs: tags: | ${{ steps.vars.outputs.BE_NAMESPACE }}/fragalysis-backend:${{ steps.vars.outputs.tag }} ${{ steps.vars.outputs.BE_NAMESPACE }}/fragalysis-backend:stable - - name: Test - run: > - docker-compose -f docker-compose.test.yml up - --build - --exit-code-from tests - --abort-on-container-exit + - name: Test (docker compose) + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: ./docker-compose.test.yml + up-flags: --build --exit-code-from tests --abort-on-container-exit env: BE_NAMESPACE: ${{ steps.vars.outputs.BE_NAMESPACE }} BE_IMAGE_TAG: ${{ steps.vars.outputs.tag }} diff --git a/.github/workflows/build-staging.yaml b/.github/workflows/build-staging.yaml index 35b70ff0..d76fe21a 100644 --- a/.github/workflows/build-staging.yaml +++ b/.github/workflows/build-staging.yaml @@ -154,12 +154,11 @@ jobs: with: context: . tags: ${{ steps.vars.outputs.BE_NAMESPACE }}/fragalysis-backend:${{ steps.vars.outputs.tag }} - - name: Test - run: > - docker-compose -f docker-compose.test.yml up - --build - --exit-code-from tests - --abort-on-container-exit + - name: Test (docker compose) + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: ./docker-compose.test.yml + up-flags: --build --exit-code-from tests --abort-on-container-exit env: BE_NAMESPACE: ${{ steps.vars.outputs.BE_NAMESPACE }} BE_IMAGE_TAG: ${{ steps.vars.outputs.tag }} diff --git a/README.md b/README.md index 808ec279..19bd9610 100644 --- a/README.md +++ b/README.md @@ -103,10 +103,16 @@ When you want to spin-down the deployment run: - docker-compose down -> When running locally (via `docker-compose`) Celery tasks are set to run - synchronously, like a function call rather than as asynchronous tasks. - This is controlled by the `CELERY_TASK_ALWAYS_EAGER` environment variable - that you'll find in the `docker-compose.yml` file. +When running locally (via `docker-compose`) Celery tasks are set to +run synchronously, like a function call rather than as asynchronous +tasks. This is controlled by the `CELERY_TASK_ALWAYS_EAGER` +environment variable that you'll find in the `docker-compose.yml` +file. If asynchronous Celery tasks are needed in local development, +they can be launched with the additional compose file: + + + docker compose -f docker-compose.yml -f docker-compose.celery.yml up + There is also a convenient bash script that can be used to build and push an image to a repository. you just need to provide the Docker image *namespace* and a *tag*. diff --git a/api/prometheus_metrics.py b/api/prometheus_metrics.py new file mode 100644 index 00000000..c2988760 --- /dev/null +++ b/api/prometheus_metrics.py @@ -0,0 +1,74 @@ +"""Prometheus metrics used by the fragalysis API module. +""" +from prometheus_client import Counter + + +class PrometheusMetrics: + """A static class to hold the Prometheus metrics for the fragalysis API module. + Each metric has its own static method to adjust it. + """ + + # Create, and initialise the metrics for this module + ssh_tunnels = Counter( + 'fragalysis_ssh_tunnels', + 'Number of SSH tunnels successfully created', + ) + ssh_tunnels.reset() + ssh_tunnel_failures = Counter( + 'fragalysis_ssh_tunnel_failures', + 'Number of SSH tunnel failures', + ) + ssh_tunnel_failures.reset() + ispyb_connections = Counter( + 'fragalysis_ispyb_connections', + 'Number of ISpyB successful connections (excluding retries)', + ) + ispyb_connections.reset() + ispyb_connection_attempts = Counter( + 'fragalysis_ispyb_connection_attempts', + 'Number of ISpyB connection retries (after initial failure)', + ) + ispyb_connection_attempts.reset() + ispyb_connection_failures = Counter( + 'fragalysis_ispyb_connection_failures', + 'Number of ISpyB connection failures', + ) + ispyb_connection_failures.reset() + proposal_cache_hit = Counter( + 'fragalysis_proposal_cache_hit', + 'Number of proposal cache hits', + ) + proposal_cache_hit.reset() + proposal_cache_miss = Counter( + 'fragalysis_proposal_cache_miss', + 'Number of proposal cache misses', + ) + proposal_cache_miss.reset() + + @staticmethod + def new_tunnel(): + PrometheusMetrics.ssh_tunnels.inc() + + @staticmethod + def failed_tunnel(): + PrometheusMetrics.ssh_tunnel_failures.inc() + + @staticmethod + def new_ispyb_connection(): + PrometheusMetrics.ispyb_connections.inc() + + @staticmethod + def new_ispyb_connection_attempt(): + PrometheusMetrics.ispyb_connection_attempts.inc() + + @staticmethod + def failed_ispyb_connection(): + PrometheusMetrics.ispyb_connection_failures.inc() + + @staticmethod + def new_proposal_cache_hit(): + PrometheusMetrics.proposal_cache_hit.inc() + + @staticmethod + def new_proposal_cache_miss(): + PrometheusMetrics.proposal_cache_miss.inc() diff --git a/api/remote_ispyb_connector.py b/api/remote_ispyb_connector.py index c27bb5a8..c410a9fe 100644 --- a/api/remote_ispyb_connector.py +++ b/api/remote_ispyb_connector.py @@ -13,6 +13,8 @@ ) from pymysql.err import OperationalError +from .prometheus_metrics import PrometheusMetrics + logger: logging.Logger = logging.getLogger(__name__) # Timeout to allow the pymysql.connect() method to connect to the DB. @@ -134,6 +136,7 @@ def remote_connect( logger.debug('Starting SSH server...') self.server.start() + PrometheusMetrics.new_tunnel() logger.debug('Started SSH server') # Try to connect to the database @@ -164,6 +167,7 @@ def remote_connect( ) logger.warning('%s', repr(oe_e)) connect_attempts += 1 + PrometheusMetrics.new_ispyb_connection_attempt() time.sleep(PYMYSQL_EXCEPTION_RECONNECT_DELAY_S) except Exception as e: if connect_attempts == 0: @@ -176,15 +180,18 @@ def remote_connect( ) logger.warning('Unexpected %s', repr(e)) connect_attempts += 1 + PrometheusMetrics.new_ispyb_connection_attempt() time.sleep(PYMYSQL_EXCEPTION_RECONNECT_DELAY_S) if self.conn is not None: if connect_attempts > 0: logger.info('Connected') + PrometheusMetrics.new_ispyb_connection() self.conn.autocommit = True else: if connect_attempts > 0: logger.info('Failed to connect') + PrometheusMetrics.failed_ispyb_connection() self.server.stop() raise ISPyBConnectionException self.last_activity_ts = time.time() diff --git a/api/security.py b/api/security.py index 8ecf6976..12cc555d 100644 --- a/api/security.py +++ b/api/security.py @@ -12,19 +12,46 @@ from django.db.models import Q from django.http import Http404, HttpResponse from ispyb.connector.mysqlsp.main import ISPyBMySQLSPConnector as Connector -from ispyb.connector.mysqlsp.main import ISPyBNoResultException +from ispyb.exception import ISPyBConnectionException, ISPyBNoResultException from rest_framework import viewsets from viewer.models import Project +from .prometheus_metrics import PrometheusMetrics from .remote_ispyb_connector import SSHConnector logger: logging.Logger = logging.getLogger(__name__) +def get_restricted_tas_user_proposal(user) -> set[str]: + """ + Used for debugging access to restricted TAS projects. + settings.RESTRICTED_TAS_USERS_LIST is a list of strings that + contain ":". We inspect this list, and if our user is in it + we collect and return them. + + This should always return an empty set() in production. + """ + assert user + + response = set() + if settings.RESTRICTED_TAS_USERS_LIST: + for item in settings.RESTRICTED_TAS_USERS_LIST: + item_username, item_tas = item.split(':') + if item_username == user.username: + response.add(item_tas) + + if response: + logger.warning( + 'Returning restricted TAS "%s" for user "%s"', item_tas, user.username + ) + return response + + @cache class CachedContent: - """A static class managing caches proposals/visits for each user. + """ + A static class managing caches proposals/visits for each user. Proposals should be collected when has_expired() returns True. Content can be written (when the cache for the user has expired) and read using the set/get methods. @@ -51,19 +78,24 @@ def has_expired(username) -> bool: has_expired = True # Expired, reset the expiry time CachedContent._timers[username] = now + CachedContent._cache_period + if has_expired: + logger.debug("Content expired for '%s'", username) return has_expired @staticmethod def get_content(username): with CachedContent._cache_lock: if username not in CachedContent._content: - CachedContent._content[username] = [] - return CachedContent._content[username] + CachedContent._content[username] = set() + content = CachedContent._content[username] + logger.debug("Got content for '%s': %s", username, content) + return content @staticmethod def set_content(username, content) -> None: with CachedContent._cache_lock: CachedContent._content[username] = content.copy() + logger.debug("Set content for '%s': %s", username, content) def get_remote_conn(force_error_display=False) -> Optional[SSHConnector]: @@ -99,14 +131,21 @@ def get_remote_conn(force_error_display=False) -> Optional[SSHConnector]: conn: Optional[SSHConnector] = None try: conn = SSHConnector(**credentials) + except ISPyBConnectionException: + # The ISPyB connection failed. + # Nothing else to do here, metrics are already updated + pass except Exception: + # Any other exception will be a problem with the SSH tunnel connection + PrometheusMetrics.failed_tunnel() if logging.DEBUG >= logger.level or force_error_display: logger.info("credentials=%s", credentials) logger.exception("Got the following exception creating Connector...") + if conn: - logger.debug("Got remote connector") + logger.debug("Got remote ISPyB connector") else: - logger.debug("Failed to get a remote connector") + logger.debug("Failed to get a remote ISPyB connector") return conn @@ -140,8 +179,10 @@ def get_conn(force_error_display=False) -> Optional[Connector]: logger.exception("Got the following exception creating Connector...") if conn: logger.debug("Got connector") + PrometheusMetrics.new_ispyb_connection() else: logger.debug("Did not get a connector") + PrometheusMetrics.failed_ispyb_connection() return conn @@ -169,10 +210,23 @@ def ping_configured_connector() -> bool: return conn is not None -class ISpyBSafeQuerySet(viewsets.ReadOnlyModelViewSet): +class ISPyBSafeQuerySet(viewsets.ReadOnlyModelViewSet): + """ + This ISpyBSafeQuerySet, which inherits from the DRF viewsets.ReadOnlyModelViewSet, + is used for all views that need to yield (filter) view objects based on a + user's proposal membership. This requires the view to define the property + "filter_permissions" to enable this class to navigate to the view object's Project + (proposal/visit). + + As the ISpyBSafeQuerySet is based on a ReadOnlyModelViewSet, which only provides + implementations for list() and retrieve() methods, the user will need to provide + "mixins" for any additional methods the view needs to support (PATCH, PUT, DELETE). + """ + def get_queryset(self): """ - Optionally restricts the returned purchases to a given proposals + Restricts the returned records to those that belong to proposals + the user has access to. Without a user only 'open' proposals are returned. """ # The list of proposals this user can have proposal_list = self.get_proposals_for_user(self.request.user) @@ -184,10 +238,10 @@ def get_queryset(self): # Must have a foreign key to a Project for this filter to work. # get_q_filter() returns a Q expression for filtering - q_filter = self.get_q_filter(proposal_list) + q_filter = self._get_q_filter(proposal_list) return self.queryset.filter(q_filter).distinct() - def _get_open_proposals(self): + def get_open_proposals(self): """ Returns the set of proposals anybody can access. These consist of any Projects that are marked "open_to_public" @@ -197,6 +251,7 @@ def _get_open_proposals(self): Project.objects.filter(open_to_public=True).values_list("title", flat=True) ) open_proposals.update(settings.PUBLIC_TAS_LIST) + # End Temporary Test Code (1247) return open_proposals def _get_proposals_for_user_from_django(self, user): @@ -231,30 +286,29 @@ def _run_query_with_connector(self, conn, user): def _get_proposals_for_user_from_ispyb(self, user): if CachedContent.has_expired(user.username): - logger.info("Cache has expired for '%s'", user.username) + PrometheusMetrics.new_proposal_cache_miss() if conn := get_configured_connector(): - logger.debug("Got a connector for '%s'", user.username) + logger.info("Got a connector for '%s'", user.username) self._get_proposals_from_connector(user, conn) else: logger.warning("Failed to get a connector for '%s'", user.username) - self._mark_cache_collection_failure(user) + else: + PrometheusMetrics.new_proposal_cache_hit() # The cache has either been updated, has not changed or is empty. # Return what we have for the user. Public (open) proposals # will be added to what we return if necessary. cached_prop_ids = CachedContent.get_content(user.username) - logger.debug( - "Have %s cached Proposals for '%s': %s", + logger.info( + "Returning %s cached Proposals for '%s'", len(cached_prop_ids), user.username, - cached_prop_ids, ) - return cached_prop_ids def _get_proposals_from_connector(self, user, conn): - """Updates the USER_LIST_DICT with the results of a query - and marks it as populated. + """ + Updates the user's proposal cache with the results of a query """ assert user assert conn @@ -302,9 +356,9 @@ def _get_proposals_from_connector(self, user, conn): proposal_visit_str = f'{proposal_str}-{sn_str}' prop_id_set.update([proposal_str, proposal_visit_str]) - # Always display the collected results for the user. + # Display the collected results for the user. # These will be cached. - logger.debug( + logger.info( "%s proposals from %s records for '%s': %s", len(prop_id_set), len(rs), @@ -313,25 +367,67 @@ def _get_proposals_from_connector(self, user, conn): ) CachedContent.set_content(user.username, prop_id_set) - def get_proposals_for_user(self, user, restrict_to_membership=False): - """Returns a list of proposals that the user has access to. + def user_is_member_of_target( + self, user, target, restrict_public_to_membership=True + ): + """ + Returns true if the user has access to any proposal the target belongs to. + """ + target_proposals = [p.title for p in target.project_id.all()] + user_proposals = self.get_proposals_for_user( + user, restrict_public_to_membership=restrict_public_to_membership + ) + is_member = any(proposal in user_proposals for proposal in target_proposals) + if not is_member: + logger.warning( + "Failed membership check user='%s' target='%s' target_proposals=%s", + user.username, + target.title, + target_proposals, + ) + return is_member - If 'restrict_to_membership' is set only those proposals/visits where the user + def user_is_member_of_any_given_proposals( + self, user, proposals, restrict_public_to_membership=True + ): + """ + Returns true if the user has access to any proposal in the given + proposals list. Only one needs to match for permission to be granted. + We 'restrict_public_to_membership' to only consider proposals the user + has explicit membership. + """ + user_proposals = self.get_proposals_for_user( + user, restrict_public_to_membership=restrict_public_to_membership + ) + is_member = any(proposal in user_proposals for proposal in proposals) + if not is_member: + logger.warning( + "Failed membership check user='%s' proposals=%s", + user.username, + proposals, + ) + return is_member + + def get_proposals_for_user(self, user, restrict_public_to_membership=False): + """ + Returns a list of proposals that the user has access to. + + If 'restrict_public_to_membership' is set only those proposals/visits where the user is a member of the visit will be returned. Otherwise the 'public' - proposals/visits will also be returned. Typically 'restrict_to_membership' is + proposals/visits will also be returned. Typically 'restrict_public_to_membership' is used for uploads/changes - this allows us to implement logic that (say) only permits explicit members of public proposals to add/load data for that - project (restrict_to_membership=True), but everyone can 'see' public data - (restrict_to_membership=False). + project (restrict_public_to_membership=True), but everyone can 'see' public data + (restrict_public_to_membership=False). """ assert user proposals = set() ispyb_user = settings.ISPYB_USER logger.debug( - "ispyb_user=%s restrict_to_membership=%s (DISABLE_RESTRICT_PROPOSALS_TO_MEMBERSHIP=%s)", + "ispyb_user=%s restrict_public_to_membership=%s (DISABLE_RESTRICT_PROPOSALS_TO_MEMBERSHIP=%s)", ispyb_user, - restrict_to_membership, + restrict_public_to_membership, settings.DISABLE_RESTRICT_PROPOSALS_TO_MEMBERSHIP, ) if ispyb_user: @@ -345,15 +441,21 @@ def get_proposals_for_user(self, user, restrict_to_membership=False): # We have all the proposals where the user has authority. # Add open/public proposals? if ( - not restrict_to_membership + not restrict_public_to_membership or settings.DISABLE_RESTRICT_PROPOSALS_TO_MEMBERSHIP ): - proposals.update(self._get_open_proposals()) + proposals.update(self.get_open_proposals()) + + # Finally, add any restricted TAS proposals the user has access to. + # It uses an environment variable to arbitrarily add proposals for a given user. + # This is a debug mechanism and should not be used in production. + # Added during debug effort for 1491. + proposals.update(get_restricted_tas_user_proposal(user)) # Return the set() as a list() return list(proposals) - def get_q_filter(self, proposal_list): + def _get_q_filter(self, proposal_list): """Returns a Q expression representing a (potentially complex) table filter.""" if self.filter_permissions: # Q-filter is based on the filter_permissions string @@ -372,9 +474,9 @@ def get_q_filter(self, proposal_list): return Q(title__in=proposal_list) | Q(open_to_public=True) -class ISpyBSafeStaticFiles: +class ISPyBSafeStaticFiles: def get_queryset(self): - query = ISpyBSafeQuerySet() + query = ISPyBSafeQuerySet() query.request = self.request query.filter_permissions = self.permission_string query.queryset = self.model.objects.filter() @@ -420,7 +522,7 @@ def get_response(self): raise Http404 from exc -class ISpyBSafeStaticFiles2(ISpyBSafeStaticFiles): +class ISPyBSafeStaticFiles2(ISPyBSafeStaticFiles): def get_response(self): logger.info("+ get_response called with: %s", self.input_string) # it wasn't working because found two objects with test file name diff --git a/api/urls.py b/api/urls.py index 8f4d38ea..93536f9e 100644 --- a/api/urls.py +++ b/api/urls.py @@ -3,20 +3,18 @@ from rest_framework.authtoken import views as drf_views from rest_framework.routers import DefaultRouter -# from xcdb import views as xchem_views from hotspots import views as hostpot_views from hypothesis import views as hypo_views from scoring import views as score_views from viewer import views as viewer_views -from xcdb import views as xcdb_views router = DefaultRouter() # Register the basic data router.register("compounds", viewer_views.CompoundView, "compounds") router.register("targets", viewer_views.TargetView, "targets") router.register("projects", viewer_views.ProjectView) -router.register("session-projects", viewer_views.SessionProjectsView) -router.register("snapshots", viewer_views.SnapshotsView) +router.register("session-projects", viewer_views.SessionProjectView) +router.register("snapshots", viewer_views.SnapshotView) router.register("action-type", viewer_views.ActionTypeView) router.register("session-actions", viewer_views.SessionActionsView) router.register("snapshot-actions", viewer_views.SnapshotActionsView) @@ -26,7 +24,7 @@ # Compounds sets router.register("compound-sets", viewer_views.ComputedSetView) router.register("compound-molecules", viewer_views.ComputedMoleculesView) -router.register("numerical-scores", viewer_views.NumericalScoresView) +router.register("numerical-scores", viewer_views.NumericalScoreValuesView) router.register("text-scores", viewer_views.TextScoresView) router.register("compound-scores", viewer_views.CompoundScoresView, "compound-scores") router.register( @@ -65,16 +63,13 @@ # Get the information router.register("siteobservationannotation", score_views.SiteObservationAnnotationView) -# fragspect -router.register("fragspect", xcdb_views.FragspectCrystalView) - # discourse posts router.register( "discourse_post", viewer_views.DiscoursePostView, basename='discourse_post' ) # Take a dictionary and return a csv -router.register("dicttocsv", viewer_views.DictToCsv, basename='dicttocsv') +router.register("dicttocsv", viewer_views.DictToCSVView, basename='dicttocsv') # tags router.register("tag_category", viewer_views.TagCategoryView, basename='tag_category') @@ -92,36 +87,38 @@ # Download a zip file of the requested contents router.register( "download_structures", - viewer_views.DownloadStructures, + viewer_views.DownloadStructuresView, basename='download_structures', ) # Experiments and Experiment (XChemAlign) upload support router.register( "upload_target_experiments", - viewer_views.UploadTargetExperiments, + viewer_views.UploadExperimentUploadView, basename='upload_target_experiments', ) router.register( "download_target_experiments", - viewer_views.DownloadTargetExperiments, + viewer_views.DownloadExperimentUploadView, basename='download_target_experiments', ) router.register( "target_experiment_uploads", - viewer_views.TargetExperimentUploads, + viewer_views.ExperimentUploadView, basename='target_experiment_uploads', ) router.register( - "site_observations", viewer_views.SiteObservations, basename='site_observations' + "site_observations", viewer_views.SiteObservationView, basename='site_observations' +) +router.register("canon_sites", viewer_views.CanonSiteView, basename='canon_sites') +router.register( + "canon_site_confs", viewer_views.CanonSiteConfView, basename='canon_site_confs' ) -router.register("canon_sites", viewer_views.CanonSites, basename='canon_sites') router.register( - "canon_site_confs", viewer_views.CanonSiteConfs, basename='canon_site_confs' + "xtalform_sites", viewer_views.XtalformSiteView, basename='xtalform_sites' ) -router.register("xtalform_sites", viewer_views.XtalformSites, basename='xtalform_sites') router.register("poses", viewer_views.PoseView, basename='poses') # Squonk Jobs diff --git a/api/utils.py b/api/utils.py index f1196349..dc1a40e0 100644 --- a/api/utils.py +++ b/api/utils.py @@ -305,20 +305,16 @@ def parse_xenons(input_smi): return bond_ids, bond_colours, e_mol.GetMol() -def get_params(smiles, request): +def get_img_from_smiles(smiles, request): # try: smiles = canon_input(smiles) # except: # smiles = "" - height = None mol = None bond_id_list = [] highlightBondColors = {} - if "height" in request.GET: - height = int(request.GET["height"]) - width = None - if "width" in request.GET: - width = int(request.GET["width"]) + height = int(request.GET.get("height", "128")) + width = int(request.GET.get("width", "128")) if "atom_indices" in request.GET: mol = Chem.MolFromSmiles(smiles) bond_id_list, highlightBondColors, mol = parse_atom_ids( @@ -360,7 +356,7 @@ def get_highlighted_diffs(request): def mol_view(request): if "smiles" in request.GET: smiles = request.GET["smiles"].rstrip(".svg") - return get_params(smiles, request) + return get_img_from_smiles(smiles, request) else: return HttpResponse("Please insert SMILES") diff --git a/docker-compose.celery.yml b/docker-compose.celery.yml new file mode 100644 index 00000000..fe8d6875 --- /dev/null +++ b/docker-compose.celery.yml @@ -0,0 +1,56 @@ +# Override compose file to enable celery services locally +# Adds containers for beat and worker + minor tweaks for main backend + +# Run like: +# sudo docker compose -f docker-compose.yml -f docker-compose.celery.yml up + +version: '3' + +services: + + # The stack backend + backend: + env_file: + - .env + environment: + # Celery tasks run as intended here + CELERY_TASK_ALWAYS_EAGER: 'False' + healthcheck: + test: python manage.py --help || exit 1 + interval: 10s + timeout: 10s + retries: 20 + start_period: 10s + + + celery_worker: + command: sh -c "celery -A fragalysis worker -l info" + container_name: celery_worker + depends_on: + database: + condition: service_healthy + redis: + condition: service_healthy + backend: + condition: service_healthy + hostname: celery_worker + env_file: + - .env + image: ${BE_NAMESPACE:-xchem}/fragalysis-backend:${BE_IMAGE_TAG:-latest} + restart: on-failure + + celery_beat: + command: sh -c "celery -A fragalysis beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" + container_name: celery_beat + depends_on: + database: + condition: service_healthy + redis: + condition: service_healthy + backend: + condition: service_healthy + hostname: celery_beat + env_file: + - .env + image: ${BE_NAMESPACE:-xchem}/fragalysis-backend:${BE_IMAGE_TAG:-latest} + restart: on-failure diff --git a/docker-compose.yml b/docker-compose.yml index adcbde8e..1e1d13f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,8 @@ services: - ./data/logs:/code/logs/ - ./data/media:/code/media/ - .:/code/ + env_file: + - .env environment: AUTHENTICATE_UPLOAD: ${AUTHENTICATE_UPLOAD:-True} DEPLOYMENT_MODE: 'development' diff --git a/docs/source/API/api_intro.rst b/docs/source/API/api_intro.rst index 152a10cc..aed8f60f 100644 --- a/docs/source/API/api_intro.rst +++ b/docs/source/API/api_intro.rst @@ -23,7 +23,7 @@ The most common operations are: A more complex overview of these methods (and others) is available here: https://www.w3schools.com/tags/ref_httpmethods.asp -A more thorough explaination of RESTful APIs is available here: https://searchapparchitecture.techtarget.com/definition/RESTful-API +A more thorough explanation of RESTful APIs is available here: https://searchapparchitecture.techtarget.com/definition/RESTful-API Django REST framework (DRF) --------------------------- @@ -50,6 +50,67 @@ user to filter by (:code:`filter_fields`) and what :code:`Response` we want to p Finally, we need to specify an :code:`endpoint` (i.e. URL) that the :code:`View` is served at, so the user can make requests against the web service. +Basic style guidance +-------------------- +All new methods (functions) in **views** _should_ inherit from classes defined in DRF. +Some methods (functions) simply accept the :code:`request`, but this is older code. + +**Security** + +When writing a new API endpoint, it is important to consider the security implications +of the data you are making available to the user. Much of the data in fragalysis is +_open_, i.e. available to all. + +.. note:: + Open data is data associated with any :code:`Project` + that has the :code:`open_to_public` flag set to :code:`True`. + +As a general policy open data is visible to anyone, even if they are not authenticated +(logged in) but there is a policy that only logged-in users can modify or create open +data. More specifically users are required to be a member of (associated with) the +:code:`Project`` the object belongs to. To this end almost all endpoints +are required to check the object's Project (the Proposal/Visit) in order to determine +whether the User is permitted access. + +.. note:: + A user is 'associated' with a :code:`Project` (aka Proposal/Visit) if the security module in the + project's :code:`api` package is able to find the user associated with the + :code:`Project` by querying an external database. In diamond + this is an **ISPyB** MySQL database external to the stack installation whose + credentials are supplied using environment variables. + +API methods that provide access to data they must ensure that the user is authenticated +and must _strive_ to ensure that the user is associated with the :code:`Project` that +the data belongs to. + +In order to check whether the user has access to the object that is being created +or altered, each method must either identify the :code:`Project` that the object belongs to, +or there has to be a navigable path from any table record that might contain "sensitive" material +to one or more records in the :code:`Project` table. +Given a :code:`Project` is is a relatively simple task to check that the user +has been given access to us using the security module as described above, and +the code in the :code:`security` module relies on this pattern. + +These actions ar simplified through the use of the :code:`ISpyBSafeQuerySet` class +to filter objects when reading and the :code:`IsObjectProposalMember` class to +check the user has access to the object when creating or altering it. These classes +rely on the definition of :code:`filter_permissions` property to direct the +search to the object's :code:`Project`. + +View classes must generally inherit from :code:`ISpyBSafeQuerySet`, +which provides automatic filtering of objects. The :code:`ISpyBSafeQuerySet` +inherits from th :code:`ReadOnlyModelViewSet` view set. If a view also needs to provide +create, update or delete actions they should also inherit an appropriate +DRF **mixin**, adding support for a method so support the functionality that is +required: - + +- :code:`mixins.CreateModelMixin` - when supporting objects (POST) +- :code:`mixins.UpdateModelMixin` - when supporting objects (PATCH) +- :code:`mixins.DestroyModelMixin` - when supporting delete (DELETE) + +For further information refer to the `mixins`_ documentation on the DRF site. + + EXAMPLE - Model, Serializer, View and URL for Target model ---------------------------------------------------------- @@ -62,34 +123,11 @@ The Target model contains information about a protein target. In django, we defi from django.db import models class Target(models.Model): - """Django model to define a Target - a protein. - - Parameters - ---------- - title: CharField - The name of the target - init_date: DateTimeField - The date the target was initiated (autofield) - project_id: ManyToManyField - Links targets to projects for authentication - uniprot_id: Charfield - Optional field where a uniprot id can be stored - metadata: FileField - Optional file upload defining metadata about the target - can be used to add custom site labels - zip_archive: FileField - Link to zip file created from targets uploaded with the loader - """ - # The title of the project_id -> userdefined title = models.CharField(unique=True, max_length=200) - # The date it was made init_date = models.DateTimeField(auto_now_add=True) - # A field to link projects and targets together project_id = models.ManyToManyField(Project) - # Indicates the uniprot_id id for the target. Is a unique key uniprot_id = models.CharField(max_length=100, null=True) - # metadatafile containing sites info for download metadata = models.FileField(upload_to="metadata/", null=True, max_length=255) - # zip archive to download uploaded data from zip_archive = models.FileField(upload_to="archive/", null=True, max_length=255) @@ -129,6 +167,31 @@ can add extra fields, and add a method to define how we get the value of the fie :code:`Serializer` we have added the :code:`template_protein` field, and defined how we get its value with :code:`get_template_protein`. +**Models** + +Model definitions should avoid inline documentation, and instead use the django +:code:`help_text` parameter to provide this information. For example, +instead of doing this: - + +.. code-block:: python + + class Target(models.Model): + # The uniprot ID id for the target. A unique key + uniprot_id = models.CharField(max_length=100, null=True) + + +Do this: - + +.. code-block:: python + + class Target(models.Model): + uniprot_id = models.CharField( + max_length=100, + null=True, + help_text="The uniprot ID id for the target. A unique key", + ) + + **View** This :code:`View` returns a list of information about a specific target, if you pass the :code:`title` parameter to the @@ -141,7 +204,8 @@ of our standard views. Additionally, in the actual code, you will notice that :code:`TargetView(viewsets.ReadOnlyModelViewSet)` is replaced by :code:`TargetView(ISpyBSafeQuerySet)`. :code:`ISpyBSafeQuerySet` is a version of :code:`viewsets.ReadOnlyModelViewSet` -that includes an authentication method specific for the deployment of fragalysis at https://fragalysis.diamond.ac.uk +that includes an authentication method that filters records based omn a user's +membership of the object's :code:`project`. .. code-block:: python @@ -150,61 +214,11 @@ that includes an authentication method specific for the deployment of fragalysis from viewer.models import Target class TargetView(viewsets.ReadOnlyModelViewSet): - """ DjagnoRF view to retrieve info about targets - - Methods - ------- - url: - api/targets - queryset: - `viewer.models.Target.objects.filter()` - filter fields: - - `viewer.models.Target.title` - ?title= - returns: JSON - - id: id of the target object - - title: name of the target - - project_id: list of the ids of the projects the target is linked to - - protein_set: list of the ids of the protein sets the target is linked to - - template_protein: the template protein displayed in fragalysis front-end for this target - - metadata: link to the metadata file for the target if it was uploaded - - zip_archive: link to the zip archive of the uploaded data - - example output: - - .. code-block:: javascript - - "results": [ - { - "id": 62, - "title": "Mpro", - "project_id": [ - 2 - ], - "protein_set": [ - 29281, - 29274, - 29259, - 29305, - ..., - ], - "template_protein": "/media/pdbs/Mpro-x10417_0_apo.pdb", - "metadata": "http://fragalysis.diamond.ac.uk/media/metadata/metadata_2FdP5OJ.csv", - "zip_archive": "http://fragalysis.diamond.ac.uk/media/targets/Mpro.zip" - } - ] - - """ queryset = Target.objects.filter() serializer_class = TargetSerializer filter_permissions = "project_id" filter_fields = ("title",) - -The docstring for this class is formatted in a way to allow a user or developer to easily read the docstring, and -understand the URL to query, how the information is queried by django, what fields can be queried against, and what -information is returned from a request against the views URL. All of the views in this documentation are written in the -same way. - **URL** Finally, we need to define where the view is served from, in context of the root (e.g. https://fragalysis.diamond.ac.uk) @@ -243,3 +257,5 @@ If we navigate to the URL :code:`/api/targets/?title=` we are This is a page automatically generated by DRF, and includes options to see what kinds of requests you can make against this endpoint. + +.. _mixins: https://www.django-rest-framework.org/tutorial/3-class-based-views/#using-mixins diff --git a/fragalysis/apps.py b/fragalysis/apps.py new file mode 100644 index 00000000..6f441559 --- /dev/null +++ b/fragalysis/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class FragalysisConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'fragalysis' diff --git a/fragalysis/schema.py b/fragalysis/schema.py index ca8f4add..4f86bd4f 100644 --- a/fragalysis/schema.py +++ b/fragalysis/schema.py @@ -5,7 +5,6 @@ import hypothesis.schema import scoring.schema import viewer.schema -import xcdb.schema class Query( @@ -14,7 +13,6 @@ class Query( hotspots.schema.Query, pandda.schema.Query, scoring.schema.Query, - xcdb.schema.Query, graphene.ObjectType, ): # This class will inherit from multiple Queries diff --git a/fragalysis/settings.py b/fragalysis/settings.py index ef3775f1..3e44409c 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -67,7 +67,7 @@ import os import sys from datetime import timedelta -from typing import List +from typing import List, Optional import sentry_sdk from sentry_sdk.integrations.celery import CeleryIntegration @@ -130,6 +130,9 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", + # 3rd + "django_celery_beat", + "django_celery_results", # My own apps "scoring", "network", @@ -138,6 +141,7 @@ "hypothesis", "hotspots", "media_serve", + "service_status.apps.ServiceStatusConfig", # The XChem database model "xchem_db", # My utility apps @@ -343,16 +347,6 @@ } } -if os.environ.get("BUILD_XCDB") == "yes": - DATABASES["xchem_db"] = { - "ENGINE": "django.db.backends.postgresql", - "NAME": os.environ.get("XCHEM_NAME", ""), - "USER": os.environ.get("XCHEM_USER", ""), - "PASSWORD": os.environ.get("XCHEM_PASSWORD", ""), - "HOST": os.environ.get("XCHEM_HOST", ""), - "PORT": os.environ.get("XCHEM_PORT", ""), - } - CHEMCENTRAL_DB_NAME = os.environ.get("CHEMCENT_DB_NAME", "UNKNOWN") if CHEMCENTRAL_DB_NAME != "UNKNOWN": DATABASES["chemcentral"] = { @@ -430,6 +424,18 @@ "filename": os.path.join(BASE_DIR, "logs/backend.log"), "formatter": "simple", }, + 'service_status': { + 'level': 'DEBUG', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': os.path.join(BASE_DIR, "logs/service_status.log"), + 'formatter': 'simple', + "maxBytes": 5_000_000, + "backupCount": 10, + }, + }, + "root": { + "level": LOGGING_FRAMEWORK_ROOT_LEVEL, + "handlers": ["console", "rotating"], }, 'loggers': { 'api.security': {'level': 'INFO'}, @@ -439,10 +445,11 @@ 'mozilla_django_oidc': {'level': 'WARNING'}, 'urllib3': {'level': 'WARNING'}, 'paramiko': {'level': 'WARNING'}, - }, - "root": { - "level": LOGGING_FRAMEWORK_ROOT_LEVEL, - "handlers": ["console", "rotating"], + 'service_status': { + 'handlers': ['service_status', 'console'], + 'level': 'DEBUG', + 'propagate': False, + }, }, } @@ -535,6 +542,16 @@ NEO4J_QUERY: str = os.environ.get("NEO4J_QUERY", "neo4j") NEO4J_AUTH: str = os.environ.get("NEO4J_AUTH", "neo4j/neo4j") +# Does it look like we're running in Kubernetes? +# If so, let's get the namespace we're in - it will provide +# useful discrimination material in log/metrics messages. +# If there is no apparent namespace the variable will be 'None'. +OUR_KUBERNETES_NAMESPACE: Optional[str] = None +_NS_FILENAME: str = '/var/run/secrets/kubernetes.io/serviceaccount/namespace' +if os.path.isfile(_NS_FILENAME): + with open(_NS_FILENAME, 'rt', encoding='utf8') as ns_file: + OUR_KUBERNETES_NAMESPACE = ns_file.read().strip() + # These flags are used in the upload_tset form as follows. # Proposal Supported | Proposal Required | Proposal / View fields # Y | Y | Shown / Required @@ -548,6 +565,14 @@ PUBLIC_TAS: str = os.environ.get("PUBLIC_TAS", "") PUBLIC_TAS_LIST: List[str] = PUBLIC_TAS.split(",") if PUBLIC_TAS else [] +# A debug mechanism to allow us to manually add user and project associations. +# The input is a comma-separated list of "user:project: pairs, +# e.g. "user-1:lb32627-66,user2:lb32627-66" +RESTRICTED_TAS_USERS: str = os.environ.get("RESTRICTED_TAS_USERS", "") +RESTRICTED_TAS_USERS_LIST: List[str] = ( + RESTRICTED_TAS_USERS.split(",") if RESTRICTED_TAS_USERS else [] +) + # Security/access control connector. # Currently one of 'ispyb' or 'ssh_ispyb'. SECURITY_CONNECTOR: str = os.environ.get("SECURITY_CONNECTOR", "ispyb").lower() @@ -603,6 +628,10 @@ TARGET_LOADER_MEDIA_DIRECTORY: str = "target_loader_data" +# A warning messages issued by the f/e. +# Used, if set, to populate the 'target_warning_message' context variable +TARGET_WARNING_MESSAGE: str = os.environ.get("TARGET_WARNING_MESSAGE", "") + # The Target Access String (TAS) Python regular expression. # The Project title (the TAS) must match this expression to be valid. # See api/utils.py validate_tas() for the current implementation. diff --git a/fragalysis/urls.py b/fragalysis/urls.py index 048f1252..2d44e47c 100644 --- a/fragalysis/urls.py +++ b/fragalysis/urls.py @@ -31,7 +31,6 @@ path("api/", include("api.urls")), path("media/", include("media_serve.urls")), path("scoring/", include("scoring.urls")), - path("xcdb/", include("xcdb.urls")), path("graphql/", GraphQLView.as_view(graphiql=True)), path('oidc/', include('mozilla_django_oidc.urls')), path( diff --git a/grafana/dashboard.json b/grafana/dashboard.json new file mode 100644 index 00000000..095d647b --- /dev/null +++ b/grafana/dashboard.json @@ -0,0 +1,562 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.5" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "The Fragalysis Stack Grafana Dashboard", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Security (API)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of hits and misses of the proposal/visit cache", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(fragalysis_proposal_cache_miss_total{namespace=\"$Namespace\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Miss", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(fragalysis_proposal_cache_hit_total{namespace=\"$Namespace\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Hit", + "range": true, + "refId": "A" + } + ], + "title": "Proposal Cache", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of SSH tunnel connection successes and failures", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(fragalysis_ssh_tunnel_failures_total{namespace=\"$Namespace\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Failure", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(fragalysis_ssh_tunnels_total{namespace=\"$Namespace\"}[$__rate_interval])", + "legendFormat": "Success", + "range": true, + "refId": "A" + } + ], + "title": "SSH Tunnels", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of ISPyB (MySQL) connection successes, failures, and retries", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(fragalysis_ispyb_connection_attempts_total{namespace=\"$Namespace\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Retry", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(fragalysis_ispyb_connection_failures_total{namespace=\"$Namespace\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Failure", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(fragalysis_ispyb_connections_total[$__rate_interval])", + "hide": false, + "legendFormat": "Success", + "range": true, + "refId": "A" + } + ], + "title": "ISPyB Connections", + "type": "timeseries" + } + ], + "refresh": "5m", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "fragalysis" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(fragalysis_ispyb_connections_total, namespace)", + "description": "The kubernetes Namespace of the Fragalysis Stack of interest", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Namespace", + "options": [], + "query": { + "query": "label_values(fragalysis_ispyb_connections_total, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Fragalysis", + "uid": "Ue0SYxPIk", + "version": 9, + "weekStart": "" +} diff --git a/hotspots/views.py b/hotspots/views.py index a76bd4f4..b7e5b762 100644 --- a/hotspots/views.py +++ b/hotspots/views.py @@ -1,10 +1,10 @@ -from rest_framework import viewsets - +from api.security import ISPyBSafeQuerySet from hotspots.models import HotspotMap from hotspots.serializers import HotspotMapSerializer -class HotspotView(viewsets.ReadOnlyModelViewSet): +class HotspotView(ISPyBSafeQuerySet): queryset = HotspotMap.objects.all() serializer_class = HotspotMapSerializer filterset_fields = ("map_type", "target", "site_observation") + filter_permissions = "target__project_id" diff --git a/hypothesis/views.py b/hypothesis/views.py index ab1557d7..1872bfa6 100644 --- a/hypothesis/views.py +++ b/hypothesis/views.py @@ -1,5 +1,4 @@ -from rest_framework import viewsets - +from api.security import ISPyBSafeQuerySet from hypothesis.models import Interaction, InteractionPoint, TargetResidue from hypothesis.serializers import ( InteractionPointSerializer, @@ -8,7 +7,7 @@ ) -class InteractionView(viewsets.ReadOnlyModelViewSet): +class InteractionView(ISPyBSafeQuerySet): queryset = Interaction.objects.filter() serializer_class = InteractionSerializer filterset_fields = ( @@ -22,15 +21,18 @@ class InteractionView(viewsets.ReadOnlyModelViewSet): "prot_smarts", "mol_smarts", ) + filter_permissions = "interaction_point__targ_res__target_id__project_id" -class InteractionPointView(viewsets.ReadOnlyModelViewSet): +class InteractionPointView(ISPyBSafeQuerySet): queryset = InteractionPoint.objects.all() serializer_class = InteractionPointSerializer filterset_fields = ("site_observation", "protein_atom_name", "molecule_atom_name") + filter_permissions = "targ_res__target_id__project_id" -class TargetResidueView(viewsets.ReadOnlyModelViewSet): +class TargetResidueView(ISPyBSafeQuerySet): queryset = TargetResidue.objects.all() serializer_class = TargetResidueSerialzier filterset_fields = ("target_id", "res_name", "res_num", "chain_id") + filter_permissions = "target_id__project_id" diff --git a/launch-stack.sh b/launch-stack.sh index 2c8298f2..86a43cf0 100755 --- a/launch-stack.sh +++ b/launch-stack.sh @@ -63,5 +63,8 @@ echo proxy_set_header X-Forwarded-Proto "${PROXY_FORWARDED_PROTO_HEADER:-https}; echo "Testing nginx config..." nginx -tq +echo "Launching service health check queries" +python manage.py start_service_queries + echo "Running nginx..." nginx diff --git a/media_serve/views.py b/media_serve/views.py index c44411fa..ce599aeb 100644 --- a/media_serve/views.py +++ b/media_serve/views.py @@ -1,6 +1,6 @@ import logging -from api.security import ISpyBSafeStaticFiles, ISpyBSafeStaticFiles2 +from api.security import ISPyBSafeStaticFiles, ISPyBSafeStaticFiles2 from viewer.models import SiteObservation, Target logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ def file_download(request, file_path): :return: the response (a redirect to nginx internal) """ logger.info("+ Received file_download file path: %s", file_path) - ispy_b_static = ISpyBSafeStaticFiles2() + ispy_b_static = ISPyBSafeStaticFiles2() # ispy_b_static = ISpyBSafeStaticFiles() ispy_b_static.model = SiteObservation ispy_b_static.request = request @@ -64,7 +64,7 @@ def tld_download(request, file_path): :return: the response (a redirect to nginx internal) """ logger.info("+ Received tld_download file path: %s", file_path) - ispy_b_static = ISpyBSafeStaticFiles2() + ispy_b_static = ISPyBSafeStaticFiles2() # ispy_b_static = ISpyBSafeStaticFiles() ispy_b_static.model = SiteObservation ispy_b_static.request = request @@ -88,7 +88,7 @@ def cspdb_download(request, file_path): :return: the response (a redirect to nginx internal) """ logger.info("+ Received cspdb_download file path: %s", file_path) - ispy_b_static = ISpyBSafeStaticFiles2() + ispy_b_static = ISPyBSafeStaticFiles2() ispy_b_static.model = SiteObservation ispy_b_static.request = request # the following 2 aren't used atm @@ -109,7 +109,7 @@ def bound_download(request, file_path): :param file_path: the file path we're getting from the static :return: the response (a redirect to nginx internal) """ - ispy_b_static = ISpyBSafeStaticFiles() + ispy_b_static = ISPyBSafeStaticFiles() ispy_b_static.model = SiteObservation ispy_b_static.request = request ispy_b_static.permission_string = "target_id__project_id" @@ -127,7 +127,7 @@ def map_download(request, file_path): :param file_path: the file path we're getting from the static :return: the response (a redirect to nginx internal) """ - ispy_b_static = ISpyBSafeStaticFiles() + ispy_b_static = ISPyBSafeStaticFiles() ispy_b_static.model = SiteObservation ispy_b_static.request = request ispy_b_static.permission_string = "target_id__project_id" @@ -163,7 +163,7 @@ def metadata_download(request, file_path): :param file_path: the file path we're getting from the static :return: the response (a redirect to nginx internal) """ - ispy_b_static = ISpyBSafeStaticFiles() + ispy_b_static = ISPyBSafeStaticFiles() ispy_b_static.model = Target ispy_b_static.request = request ispy_b_static.permission_string = "project_id" @@ -181,7 +181,7 @@ def archive_download(request, file_path): :param file_path: the file path we're getting from the static :return: the response (a redirect to nginx internal) """ - ispy_b_static = ISpyBSafeStaticFiles() + ispy_b_static = ISPyBSafeStaticFiles() ispy_b_static.model = Target ispy_b_static.request = request ispy_b_static.permission_string = "project_id" diff --git a/poetry.lock b/poetry.lock index 4bcad13d..cb36caa8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -44,13 +44,13 @@ tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] [[package]] name = "astroid" -version = "3.2.1" +version = "3.2.2" description = "An abstract syntax tree for Python with inference support." optional = false python-versions = ">=3.8.0" files = [ - {file = "astroid-3.2.1-py3-none-any.whl", hash = "sha256:b452064132234819f023b94f4bd045b250ea0009f372b4377cfcd87f10806ca5"}, - {file = "astroid-3.2.1.tar.gz", hash = "sha256:902564b36796ba1eab3ad2c7a694861fbd926f574d5dbb5fa1d86778a2ba2d91"}, + {file = "astroid-3.2.2-py3-none-any.whl", hash = "sha256:e8a0083b4bb28fcffb6207a3bfc9e5d0a68be951dd7e336d5dcf639c682388c0"}, + {file = "astroid-3.2.2.tar.gz", hash = "sha256:8ead48e31b92b2e217b6c9733a21afafe479d52d6e164dd25fb1a770c7c3cf94"}, ] [[package]] @@ -520,6 +520,20 @@ files = [ [package.dependencies] jinja2 = "*" +[[package]] +name = "cron-descriptor" +version = "1.4.3" +description = "A Python library that converts cron expressions into human readable strings." +optional = false +python-versions = "*" +files = [ + {file = "cron_descriptor-1.4.3-py3-none-any.whl", hash = "sha256:a67ba21804983b1427ed7f3e1ec27ee77bf24c652b0430239c268c5ddfbf9dc0"}, + {file = "cron_descriptor-1.4.3.tar.gz", hash = "sha256:7b1a00d7d25d6ae6896c0da4457e790b98cba778398a3d48e341e5e0d33f0488"}, +] + +[package.extras] +dev = ["polib"] + [[package]] name = "cryptography" version = "42.0.7" @@ -674,6 +688,39 @@ files = [ [package.dependencies] Django = ">=3.2" +[[package]] +name = "django-celery-beat" +version = "2.6.0" +description = "Database-backed Periodic Tasks." +optional = false +python-versions = "*" +files = [ + {file = "django-celery-beat-2.6.0.tar.gz", hash = "sha256:f75b2d129731f1214be8383e18fae6bfeacdb55dffb2116ce849222c0106f9ad"}, +] + +[package.dependencies] +celery = ">=5.2.3,<6.0" +cron-descriptor = ">=1.2.32" +Django = ">=2.2,<5.1" +django-timezone-field = ">=5.0" +python-crontab = ">=2.3.4" +tzdata = "*" + +[[package]] +name = "django-celery-results" +version = "2.5.1" +description = "Celery result backends for Django." +optional = false +python-versions = "*" +files = [ + {file = "django_celery_results-2.5.1-py3-none-any.whl", hash = "sha256:0da4cd5ecc049333e4524a23fcfc3460dfae91aa0a60f1fae4b6b2889c254e01"}, + {file = "django_celery_results-2.5.1.tar.gz", hash = "sha256:3ecb7147f773f34d0381bac6246337ce4cf88a2ea7b82774ed48e518b67bb8fd"}, +] + +[package.dependencies] +celery = ">=5.2.7,<6.0" +Django = ">=3.2.18" + [[package]] name = "django-cleanup" version = "8.1.0" @@ -772,6 +819,20 @@ files = [ [package.dependencies] asgiref = ">=3.6" +[[package]] +name = "django-timezone-field" +version = "6.1.0" +description = "A Django app providing DB, form, and REST framework fields for zoneinfo and pytz timezone objects." +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "django_timezone_field-6.1.0-py3-none-any.whl", hash = "sha256:0095f43da716552fcc606783cfb42cb025892514f1ec660ebfa96186eb83b74c"}, + {file = "django_timezone_field-6.1.0.tar.gz", hash = "sha256:d40f7059d7bae4075725d04a9dae601af9fe3c7f0119a69b0e2c6194a782f797"}, +] + +[package.dependencies] +Django = ">=3.2,<6.0" + [[package]] name = "django-webpack-loader" version = "0.7.0" @@ -2169,17 +2230,17 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pylint" -version = "3.2.0" +version = "3.2.2" description = "python code static checker" optional = false python-versions = ">=3.8.0" files = [ - {file = "pylint-3.2.0-py3-none-any.whl", hash = "sha256:9f20c05398520474dac03d7abb21ab93181f91d4c110e1e0b32bc0d016c34fa4"}, - {file = "pylint-3.2.0.tar.gz", hash = "sha256:ad8baf17c8ea5502f23ae38d7c1b7ec78bd865ce34af9a0b986282e2611a8ff2"}, + {file = "pylint-3.2.2-py3-none-any.whl", hash = "sha256:3f8788ab20bb8383e06dd2233e50f8e08949cfd9574804564803441a4946eab4"}, + {file = "pylint-3.2.2.tar.gz", hash = "sha256:d068ca1dfd735fb92a07d33cb8f288adc0f6bc1287a139ca2425366f7cbe38f8"}, ] [package.dependencies] -astroid = ">=3.2.0,<=3.3.0-dev0" +astroid = ">=3.2.2,<=3.3.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, @@ -2228,13 +2289,13 @@ pylint = ">=1.7" [[package]] name = "pymysql" -version = "1.1.0" +version = "1.1.1" description = "Pure Python MySQL Driver" optional = false python-versions = ">=3.7" files = [ - {file = "PyMySQL-1.1.0-py3-none-any.whl", hash = "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7"}, - {file = "PyMySQL-1.1.0.tar.gz", hash = "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96"}, + {file = "PyMySQL-1.1.1-py3-none-any.whl", hash = "sha256:4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c"}, + {file = "pymysql-1.1.1.tar.gz", hash = "sha256:e127611aaf2b417403c60bf4dc570124aeb4a57f5f37b8e95ae399a42f904cd0"}, ] [package.extras] @@ -2297,6 +2358,23 @@ files = [ {file = "PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"}, ] +[[package]] +name = "python-crontab" +version = "3.1.0" +description = "Python Crontab API" +optional = false +python-versions = "*" +files = [ + {file = "python-crontab-3.1.0.tar.gz", hash = "sha256:f4ea1605d24533b67fa7a634ef26cb59a5f2e7954f6e677d2d7a2229959a2fc8"}, +] + +[package.dependencies] +python-dateutil = "*" + +[package.extras] +cron-description = ["cron-descriptor"] +cron-schedule = ["croniter"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2357,7 +2435,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2365,16 +2442,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2391,7 +2460,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2399,7 +2467,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2459,13 +2526,13 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)" [[package]] name = "requests" -version = "2.31.0" +version = "2.32.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -2560,19 +2627,18 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "69.5.1" +version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, - {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "shortuuid" @@ -3018,4 +3084,4 @@ test = ["coverage"] [metadata] lock-version = "2.0" python-versions = "^3.11.3" -content-hash = "f5ab769523c6d0b223a4020f92f5929d919472c348a5d08767a88f3570e85e8f" +content-hash = "b4b37933c6d486c4fd36e6224b1173e5a356868c6779fe23487784300419123f" diff --git a/pyproject.toml b/pyproject.toml index ca0f367e..d7efcab2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ xchem-db = "0.1.26b0" djangorestframework = "3.14.0" # Less strict (flexible dependency) packages... -celery = "^5.3.1" +celery = "^5.4" deepdiff = "^6.2.0" django-bootstrap3 = "^23.4" django-cleanup = "^8.0.0" @@ -47,6 +47,8 @@ shortuuid = "^1.0.11" sshtunnel = "^0.4.0" urllib3 = "^2.0.4" validators = "^0.20.0" +django-celery-beat = "^2.6.0" +django-celery-results = "^2.5.1" # Blocked packages... # diff --git a/scoring/managers.py b/scoring/managers.py index cb2bd8b0..40cbdd5e 100644 --- a/scoring/managers.py +++ b/scoring/managers.py @@ -30,3 +30,106 @@ def filter_qs(self): def by_target(self, target): return self.get_queryset().filter_qs().filter(target=target.id) + + +class SiteObservationChoiceQueryset(QuerySet): + def filter_qs(self): + SiteObservationChoice = apps.get_model("scoring", "SiteObservationChoice") + qs = SiteObservationChoice.objects.prefetch_related( + "site_observation", + "site_observation__experiment", + "site_observation__experiment__experiment_upload", + "site_observation__experiment__experiment_upload__target", + ).annotate( + target=F("site_observation__experiment__experiment_upload__target"), + ) + + return qs + + +class SiteObservationChoiceDataManager(Manager): + def get_queryset(self): + return SiteObservationChoiceQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class CmpdChoiceQueryset(QuerySet): + def filter_qs(self): + CmpdChoice = apps.get_model("scoring", "CmpdChoice") + qs = CmpdChoice.objects.prefetch_related( + "cmpd_id__siteobservation", + "cmpd_id__siteobservation__experiment", + "cmpd_id__siteobservation__experiment__experiment_upload", + "cmpd_id__siteobservation__experiment__experiment_upload__target", + ).annotate( + target=F("cmpd_id__siteobservation__experiment__experiment_upload__target"), + ) + + return qs + + +class CmpdChoiceDataManager(Manager): + def get_queryset(self): + return CmpdChoiceQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class ViewSceneQueryset(QuerySet): + def filter_qs(self): + ViewScene = apps.get_model("scoring", "ViewScene") + qs = ViewScene.objects.prefetch_related( + "snapshot__session_project__target", + ).annotate( + target=F("snapshot__session_project__target"), + ) + + return qs + + +class ViewSceneDataManager(Manager): + def get_queryset(self): + return ViewSceneQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class SiteObservationAnnotationQueryset(QuerySet): + def filter_qs(self): + SiteObservationAnnotation = apps.get_model( + "scoring", "SiteObservationAnnotation" + ) + qs = SiteObservationAnnotation.objects.prefetch_related( + "site_observation", + "site_observation__experiment", + "site_observation__experiment__experiment_upload", + "site_observation__experiment__experiment_upload__target", + ).annotate( + target=F("site_observation__experiment__experiment_upload__target"), + ) + + return qs + + +class SiteObservationAnnotationDataManager(Manager): + def get_queryset(self): + return SiteObservationChoiceQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) diff --git a/scoring/models.py b/scoring/models.py index 39483897..dbaf42fe 100644 --- a/scoring/models.py +++ b/scoring/models.py @@ -3,7 +3,13 @@ from viewer.models import Compound, SiteObservation, Snapshot, Target -from .managers import ScoreChoiceDataManager +from .managers import ( + CmpdChoiceDataManager, + ScoreChoiceDataManager, + SiteObservationAnnotationDataManager, + SiteObservationChoiceDataManager, + ViewSceneDataManager, +) class ViewScene(models.Model): @@ -29,6 +35,9 @@ class ViewScene(models.Model): # for redirecting to project's snapshot snapshot = models.ForeignKey(Snapshot, null=True, on_delete=models.CASCADE) + objects = models.Manager() + filter_manager = ViewSceneDataManager() + class SiteObservationChoice(models.Model): """ @@ -50,6 +59,9 @@ class SiteObservationChoice(models.Model): # Score - score = models.FloatField(null=True) + objects = models.Manager() + filter_manager = SiteObservationChoiceDataManager() + class Meta: constraints = [ models.UniqueConstraint( @@ -68,6 +80,9 @@ class SiteObservationAnnotation(models.Model): annotation_type = models.CharField(max_length=50) annotation_text = models.CharField(max_length=100) + objects = models.Manager() + filter_manager = SiteObservationAnnotationDataManager() + class Meta: constraints = [ models.UniqueConstraint( @@ -128,6 +143,9 @@ class CmpdChoice(models.Model): # E.g. score = models.FloatField(null=True) + objects = models.Manager() + filter_manager = CmpdChoiceDataManager() + class Meta: unique_together = ("user_id", "cmpd_id", "choice_type") diff --git a/scoring/serializers.py b/scoring/serializers.py index 0bd21b67..3400fb58 100644 --- a/scoring/serializers.py +++ b/scoring/serializers.py @@ -8,9 +8,10 @@ SiteObservationGroup, ViewScene, ) +from viewer.serializers import ValidateProjectMixin -class ViewSceneSerializer(serializers.ModelSerializer): +class ViewSceneSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = ViewScene fields = ( @@ -25,25 +26,46 @@ class Meta: ) -class SiteObservationChoiceSerializer(serializers.ModelSerializer): +class SiteObservationChoiceSerializer( + ValidateProjectMixin, serializers.ModelSerializer +): class Meta: model = SiteObservationChoice - fields = ("id", "user", "site_observation", "choice_type", "score") + fields = ( + "id", + "user", + "site_observation", + "choice_type", + "score", + ) -class SiteObservationAnnotationSerializer(serializers.ModelSerializer): +class SiteObservationAnnotationSerializer( + ValidateProjectMixin, serializers.ModelSerializer +): class Meta: model = SiteObservationAnnotation - fields = ("id", "site_observation", "annotation_type", "annotation_text") + fields = ( + "id", + "site_observation", + "annotation_type", + "annotation_text", + ) -class CmpdChoiceSerializer(serializers.ModelSerializer): +class CmpdChoiceSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = CmpdChoice - fields = ("id", "user_id", "cmpd_id", "choice_type", "score") + fields = ( + "id", + "user_id", + "cmpd_id", + "choice_type", + "score", + ) -class ScoreChoiceSerializer(serializers.ModelSerializer): +class ScoreChoiceSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = ScoreChoice fields = ( @@ -56,7 +78,7 @@ class Meta: ) -class SiteObservationGroupSerializer(serializers.ModelSerializer): +class SiteObservationGroupSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = SiteObservationGroup fields = ( diff --git a/scoring/views.py b/scoring/views.py index af1d556e..1706b0a8 100644 --- a/scoring/views.py +++ b/scoring/views.py @@ -2,8 +2,9 @@ from django.http import HttpResponse from frag.conf.functions import generate_confs_for_vector -from rest_framework import viewsets +from rest_framework import mixins +from api.security import ISPyBSafeQuerySet from scoring.models import ( CmpdChoice, ScoreChoice, @@ -20,20 +21,33 @@ SiteObservationGroupSerializer, ViewSceneSerializer, ) +from viewer.permissions import IsObjectProposalMember -class ViewSceneView(viewsets.ModelViewSet): - queryset = ViewScene.objects.all().order_by("-modified") +class ViewSceneView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): + queryset = ViewScene.filter_manager.filter_qs().order_by("-modified") # filter_backends = (filters.DjangoFilterBackend,) serializer_class = ViewSceneSerializer filterset_fields = ("user_id", "uuid") + filter_permissions = "snapshot__session_project__target__project_id" + permission_classes = [IsObjectProposalMember] def put(self, request, *args, **kwargs): return self.partial_update(request, *args, **kwargs) -class SiteObservationChoiceView(viewsets.ModelViewSet): - queryset = SiteObservationChoice.objects.all() +class SiteObservationChoiceView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): + queryset = SiteObservationChoice.filter_manager.filter_qs() serializer_class = SiteObservationChoiceSerializer filterset_fields = ( "user", @@ -41,21 +55,42 @@ class SiteObservationChoiceView(viewsets.ModelViewSet): "site_observation__experiment__experiment_upload__target", "choice_type", ) + filter_permissions = "site_observation__experiment__experiment_upload__project" + permission_classes = [IsObjectProposalMember] -class SiteObservationAnnotationView(viewsets.ModelViewSet): - queryset = SiteObservationAnnotation.objects.all() +class SiteObservationAnnotationView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): + queryset = SiteObservationAnnotation.filter_manager.filter_qs() serializer_class = SiteObservationAnnotationSerializer filterset_fields = ("site_observation", "annotation_type") + filter_permissions = "site_observation__experiment__experiment_upload__project" + permission_classes = [IsObjectProposalMember] -class CmpdChoiceView(viewsets.ModelViewSet): - queryset = CmpdChoice.objects.all() +class CmpdChoiceView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): + queryset = CmpdChoice.filter_manager.filter_qs() serializer_class = CmpdChoiceSerializer filterset_fields = ("user_id", "cmpd_id", "choice_type") + filter_permissions = "cmpd_id__project_id" + permission_classes = [IsObjectProposalMember] -class ScoreChoiceView(viewsets.ModelViewSet): +class ScoreChoiceView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): queryset = ScoreChoice.filter_manager.filter_qs() serializer_class = ScoreChoiceSerializer filterset_fields = ( @@ -65,12 +100,21 @@ class ScoreChoiceView(viewsets.ModelViewSet): "site_observation__experiment__experiment_upload__target", "choice_type", ) + filter_permissions = "site_observation__experiment__experiment_upload__project" + permission_classes = [IsObjectProposalMember] -class SiteObservationGroupView(viewsets.ModelViewSet): +class SiteObservationGroupView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): queryset = SiteObservationGroup.objects.all() serializer_class = SiteObservationGroupSerializer filterset_fields = ("group_type", "site_observation", "target", "description") + filter_permissions = "target__project_id" + permission_classes = [IsObjectProposalMember] def gen_conf_from_vect(request): diff --git a/xcdb/__init__.py b/service_status/__init__.py similarity index 100% rename from xcdb/__init__.py rename to service_status/__init__.py diff --git a/service_status/apps.py b/service_status/apps.py new file mode 100644 index 00000000..9f2fde36 --- /dev/null +++ b/service_status/apps.py @@ -0,0 +1,10 @@ +from django.apps import AppConfig + + +class ServiceStatusConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'service_status' + + def ready(self): + # dummy import needed because otherwise tasks aren't being registered + import service_status.services # pylint: disable=unused-import diff --git a/service_status/management/commands/services.py b/service_status/management/commands/services.py new file mode 100644 index 00000000..d77b8271 --- /dev/null +++ b/service_status/management/commands/services.py @@ -0,0 +1,30 @@ +from django.core.management.base import BaseCommand + +from service_status.utils import services + + +class Command(BaseCommand): + help = "Activate/deactivate service health check queries defined in service_status/services.py" + + def add_arguments(self, parser): + parser.add_argument( + "--enable", metavar="Service IDs", nargs="*", help="Enable service queries" + ) + parser.add_argument( + "--disable", + metavar="Service IDs", + nargs="*", + help="Disable service queries", + ) + + def handle(self, *args, **kwargs): + # Unused args + del args + + if "enable" not in kwargs.keys() and "disable" not in kwargs.keys(): + self.stdout.write( + self.style.ERROR("One of '--enable' or '--disable' must be defined'") + ) + return + + services(enable=kwargs["enable"], disable=kwargs["disable"]) diff --git a/service_status/management/commands/start_service_queries.py b/service_status/management/commands/start_service_queries.py new file mode 100644 index 00000000..ed241638 --- /dev/null +++ b/service_status/management/commands/start_service_queries.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand + +from service_status.utils import init_services + + +class Command(BaseCommand): + help = "Activate service health check queries on startup" + + def handle(self, *args, **kwargs): + # Unused args + del args, kwargs + init_services() diff --git a/service_status/managers.py b/service_status/managers.py new file mode 100644 index 00000000..2427178f --- /dev/null +++ b/service_status/managers.py @@ -0,0 +1,23 @@ +from django.apps import apps +from django.db.models import F, Manager, QuerySet + + +class ServiceStateQueryset(QuerySet): + def to_frontend(self): + Service = apps.get_model("service_status", "Service") + + qs = Service.objects.annotate( + id=F("service"), + name=F("display_name"), + state=F("last_state"), + ).order_by("service") + + return qs + + +class ServiceStateDataManager(Manager): + def get_queryset(self): + return ServiceStateQueryset(self.model, using=self._db) + + def to_frontend(self): + return self.get_queryset().to_frontend().values("id", "name", "state") diff --git a/service_status/migrations/0001_initial.py b/service_status/migrations/0001_initial.py new file mode 100644 index 00000000..bc7a8f4f --- /dev/null +++ b/service_status/migrations/0001_initial.py @@ -0,0 +1,36 @@ +# Generated by Django 3.2.25 on 2024-05-17 13:54 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ServiceState', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('state', models.TextField()), + ('display_name', models.TextField()), + ], + ), + migrations.CreateModel( + name='Services', + fields=[ + ('service', models.TextField(primary_key=True, serialize=False)), + ('display_name', models.TextField()), + ('frequency', models.PositiveSmallIntegerField(default=30, help_text='Ping frequency in seconds')), + ('last_states_of_same_type', models.IntegerField(null=True)), + ('last_query_time', models.DateTimeField(null=True)), + ('last_success', models.DateTimeField(null=True)), + ('last_failure', models.DateTimeField(null=True)), + ('last_state', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='service_status.servicestate')), + ], + ), + ] diff --git a/service_status/migrations/0002_auto_20240517_1356.py b/service_status/migrations/0002_auto_20240517_1356.py new file mode 100644 index 00000000..c12225a5 --- /dev/null +++ b/service_status/migrations/0002_auto_20240517_1356.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.25 on 2024-05-17 13:56 + +from sys import displayhook +from django.db import migrations + +states = ['NOT_CONFIGURED', 'DEGRADED', 'OK', 'ERROR'] + +def populate_states(apps, schema_editor): + ServiceState = apps.get_model('service_status', 'ServiceState') + for state in states: + try: + _ = ServiceState.objects.get(state=state) + except ServiceState.DoesNotExist: + ServiceState( + state=state, + display_name=state.lower().replace('_', ' '), + ).save() + +def drop_states(apps, schema_editor): + ServiceState = apps.get_model('service_status', 'ServiceState') + ServiceState.objects.all().delete() + +class Migration(migrations.Migration): + + dependencies = [ + ('service_status', '0001_initial'), + ] + + operations = [ + migrations.RunPython(populate_states, drop_states), + ] diff --git a/service_status/migrations/0003_rename_services_service.py b/service_status/migrations/0003_rename_services_service.py new file mode 100644 index 00000000..7a26926f --- /dev/null +++ b/service_status/migrations/0003_rename_services_service.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.25 on 2024-05-17 14:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('service_status', '0002_auto_20240517_1356'), + ] + + operations = [ + migrations.RenameModel( + old_name='Services', + new_name='Service', + ), + ] diff --git a/service_status/migrations/0004_auto_20240517_1510.py b/service_status/migrations/0004_auto_20240517_1510.py new file mode 100644 index 00000000..1a49d10a --- /dev/null +++ b/service_status/migrations/0004_auto_20240517_1510.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.25 on 2024-05-17 15:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service_status', '0003_rename_services_service'), + ] + + operations = [ + migrations.RemoveField( + model_name='servicestate', + name='id', + ), + migrations.AlterField( + model_name='servicestate', + name='state', + field=models.TextField(primary_key=True, serialize=False), + ), + ] diff --git a/service_status/migrations/0005_remove_service_last_state.py b/service_status/migrations/0005_remove_service_last_state.py new file mode 100644 index 00000000..ec2a15b2 --- /dev/null +++ b/service_status/migrations/0005_remove_service_last_state.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.25 on 2024-05-17 15:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('service_status', '0004_auto_20240517_1510'), + ] + + operations = [ + migrations.RemoveField( + model_name='service', + name='last_state', + ), + ] diff --git a/service_status/migrations/0006_service_last_state.py b/service_status/migrations/0006_service_last_state.py new file mode 100644 index 00000000..674017bb --- /dev/null +++ b/service_status/migrations/0006_service_last_state.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-05-17 15:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('service_status', '0005_remove_service_last_state'), + ] + + operations = [ + migrations.AddField( + model_name='service', + name='last_state', + field=models.ForeignKey(default='NOT_CONFIGURED', on_delete=django.db.models.deletion.PROTECT, to='service_status.servicestate'), + ), + ] diff --git a/service_status/migrations/0007_alter_service_last_states_of_same_type.py b/service_status/migrations/0007_alter_service_last_states_of_same_type.py new file mode 100644 index 00000000..d0a85983 --- /dev/null +++ b/service_status/migrations/0007_alter_service_last_states_of_same_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-05-22 14:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service_status', '0006_service_last_state'), + ] + + operations = [ + migrations.AlterField( + model_name='service', + name='last_states_of_same_type', + field=models.IntegerField(default=0), + ), + ] diff --git a/service_status/migrations/0008_service_total_queries.py b/service_status/migrations/0008_service_total_queries.py new file mode 100644 index 00000000..4f1eac20 --- /dev/null +++ b/service_status/migrations/0008_service_total_queries.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-05-23 07:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service_status', '0007_alter_service_last_states_of_same_type'), + ] + + operations = [ + migrations.AddField( + model_name='service', + name='total_queries', + field=models.IntegerField(default=0), + ), + ] diff --git a/service_status/migrations/__init__.py b/service_status/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/service_status/models.py b/service_status/models.py new file mode 100644 index 00000000..6f9c02ed --- /dev/null +++ b/service_status/models.py @@ -0,0 +1,48 @@ +import logging + +from django.db import models + +from .managers import ServiceStateDataManager + +# TODO: separate file +logger = logging.getLogger(__name__) + + +class ServiceState(models.Model): + """Query result choices: + - NOT_CONFIGURED + - DEGRADED + - OK + - ERROR + """ + + state = models.TextField(null=False, primary_key=True) + display_name = models.TextField(null=False) + + def is_success(self) -> bool: + return self.state == 'OK' + + +class Service(models.Model): + service = models.TextField(null=False, primary_key=True) + display_name = models.TextField(null=False) + frequency = models.PositiveSmallIntegerField( + default=30, help_text='Ping frequency in seconds' + ) + last_state = models.ForeignKey( + ServiceState, null=False, on_delete=models.PROTECT, default='NOT_CONFIGURED' + ) + last_states_of_same_type = models.IntegerField(null=False, default=0) + total_queries = models.IntegerField(null=False, default=0) + last_query_time = models.DateTimeField(null=True) + last_success = models.DateTimeField(null=True) + last_failure = models.DateTimeField(null=True) + + objects = models.Manager() + data_manager = ServiceStateDataManager() + + def __str__(self) -> str: + return f"{self.service}" + + def __repr__(self) -> str: + return "" % (self.service, self.last_result) diff --git a/service_status/services.py b/service_status/services.py new file mode 100644 index 00000000..7cf2fad3 --- /dev/null +++ b/service_status/services.py @@ -0,0 +1,113 @@ +import logging +import time +from random import random + +import requests +from celery import shared_task +from django.conf import settings +from frag.utils.network_utils import get_driver +from pydiscourse import DiscourseClient + +from api.security import ping_configured_connector +from viewer.squonk2_agent import get_squonk2_agent + +from .utils import State, service_query + +logger = logging.getLogger('service_status') + + +# Default timeout for any request calls +# Used for keycloak atm. +REQUEST_TIMEOUT_S = 5 + +# Service query timeout +SERVICE_QUERY_TIMEOUT_S = 28 + + +# service status test functions +# NB! first line of docstring is used as a display name + + +# @shared_task(soft_time_limit=SERVICE_QUERY_TIMEOUT_S) +# @service_query +def test_query() -> str: + """A dumb little test query. + + For testing. + """ + logger.debug('+ test_query') + state = State.DEGRADED + time.sleep(3) + if random() > 0.2: + state = State.ERROR + if random() > 0.2: + state = State.OK + else: + state = State.ERROR + + logger.debug('end state: %s', state) + return state.name + + +@shared_task(soft_time_limit=SERVICE_QUERY_TIMEOUT_S) +@service_query +def ispyb() -> str: + """Access control (ISPyB)""" + logger.debug("+ ispyb") + return State.OK if ping_configured_connector() else State.DEGRADED + + +@shared_task(soft_time_limit=SERVICE_QUERY_TIMEOUT_S) +@service_query +def discourse() -> str: + """Discourse""" + logger.debug("+ discourse") + # Discourse is "unconfigured" if there is no API key + if not any( + [settings.DISCOURSE_API_KEY, settings.DISCOURSE_HOST, settings.DISCOURSE_USER] + ): + return State.NOT_CONFIGURED + client = DiscourseClient( + settings.DISCOURSE_HOST, + api_username=settings.DISCOURSE_USER, + api_key=settings.DISCOURSE_API_KEY, + ) + # TODO: some action on client? + return State.DEGRADED if client is None else State.OK + + +@shared_task(soft_time_limit=SERVICE_QUERY_TIMEOUT_S) +@service_query +def squonk() -> str: + """Squonk""" + logger.debug("+ squonk") + return State.OK if get_squonk2_agent().configured().success else State.DEGRADED + + +@shared_task(soft_time_limit=SERVICE_QUERY_TIMEOUT_S) +@service_query +def fragmentation_graph() -> str: + """Fragmentation graph""" + logger.debug("+ fragmentation_graph") + graph_driver = get_driver(url=settings.NEO4J_QUERY, neo4j_auth=settings.NEO4J_AUTH) + with graph_driver.session() as session: + try: + _ = session.run("match (n) return count (n);") + return State.OK + except ValueError: + # service isn't running + return State.DEGRADED + + +@shared_task(soft_time_limit=SERVICE_QUERY_TIMEOUT_S) +@service_query +def keycloak() -> str: + """Keycloak""" + logger.debug("+ keycloak") + # Keycloak is "unconfigured" if there is no realm URL + keycloak_realm = settings.OIDC_KEYCLOAK_REALM + if not keycloak_realm: + return State.NOT_CONFIGURED + response = requests.get(keycloak_realm, timeout=REQUEST_TIMEOUT_S) + logger.debug("keycloak response: %s", response) + return State.OK if response.ok else State.DEGRADED diff --git a/service_status/utils.py b/service_status/utils.py new file mode 100644 index 00000000..aa45f054 --- /dev/null +++ b/service_status/utils.py @@ -0,0 +1,193 @@ +import functools +import inspect +import logging +from enum import Enum + +from celery.exceptions import SoftTimeLimitExceeded +from django.conf import settings +from django.utils import timezone + +from fragalysis.celery import app as celery_app + +from .models import Service, ServiceState + +logger = logging.getLogger('service_status') + + +# this is a bit redundant because they're all in database, but it's +# convenient to have them here +class State(str, Enum): + NOT_CONFIGURED = "NOT_CONFIGURED" + DEGRADED = "DEGRADED" + OK = "OK" + ERROR = "ERROR" + + +def service_query(func): + """Decorator function for service queries functions""" + + @functools.wraps(func) + def wrapper_service_query(*args, **kwargs): # pylint: disable=unused-argument + import service_status.services as services_module + + try: + state_pk = func() + except SoftTimeLimitExceeded: + logger.warning('Query time limit exceeded, setting result as DEGRADED') + state_pk = State.DEGRADED + + service = Service.objects.get(service=func.__name__) + + state = ServiceState.objects.get(state=state_pk) + if service.last_state == state: + service.last_states_of_same_type = service.last_states_of_same_type + 1 + else: + service.last_states_of_same_type = 0 + + service.last_state = state + timestamp = timezone.now() + service.last_query_time = timestamp + if state.is_success(): + service.last_success = timestamp + else: + service.last_failure = timestamp + + service.total_queries = service.total_queries + 1 + + # unexplored possibility to adjust ping times if necessary + + service.save() + + # get the task function from this module + task = getattr(services_module, func.__name__) + task.apply_async(countdown=service.frequency) + + return wrapper_service_query + + +def init_services(): + logger.debug('+ init_services') + service_string = settings.ENABLE_SERVICE_STATUS + requested_services = [k for k in service_string.split(":") if k != ""] + + import service_status.services as services_module + + # gather all test functions from services.py and make sure they're + # in db + defined = [] + for name, body in inspect.getmembers(services_module): + # doesn't seem to have a neat way to test if object is task, + # have to check them manually + try: + src = inspect.getsource(body) + except TypeError: + # uninteresting propery + continue + + if src.find('@shared_task') >= 0 and src.find('@service_query') >= 0: + defined.append(name) + # ensure all defined services are in db + try: + service = Service.objects.get(service=name) + except Service.DoesNotExist: + # add if missing + docs = inspect.getdoc(body) + display_name = docs.splitlines()[0] if docs else '' + Service( + service=name, + # use first line of docstring as user-friendly name + display_name=display_name, + ).save() + + # clear up db of those that are not defined + for service in Service.objects.all(): + if service.service not in defined: + service.delete() + + # mark those not requested as NOT_CONFIGURED + for service in Service.objects.all(): + if service.service not in requested_services: + service.last_state = ServiceState.objects.get(state=State.NOT_CONFIGURED) + service.save() + + # and launch the rest + # TODO: this could potentially be an actual check if beat is running + if not settings.CELERY_TASK_ALWAYS_EAGER: + for s in requested_services: + logger.debug('trying to launch service: %s', s) + try: + service = Service.objects.get(service=s) + except Service.DoesNotExist: + logger.error( + 'Service %s requested but test function missing in services.py', + s, + ) + continue + + # launch query task + task = getattr(services_module, service.service) + logger.debug('trying to launch task: %s', task) + task.delay() + + +def services(enable=(), disable=()): + logger.debug('+ init_services') + import service_status.services as services_module + + if enable is None: + enable = [] + if disable is None: + disable = [] + + to_enable = set(enable).difference(set(disable)) + to_disable = set(disable).difference(set(enable)) + confusables = set(disable).intersection(set(enable)) + + # at this point, all the services must be started and in db + for name in to_enable: + try: + service = Service.objects.get(service=name) + except Service.DoesNotExist: + logger.error('Unknown service: %s', name) + continue + + task = getattr(services_module, service.service) + task.delay() + logger.info('Starting service query %s', name) + + inquisitor = celery_app.control.inspect() + for name in to_disable: + try: + service = Service.objects.get(service=name) + except Service.DoesNotExist: + logger.error('Unknown service: %s', name) + continue + + task = getattr(services_module, service.service) + for tasklist in inquisitor.active().values(): + for worker_task in tasklist: + if worker_task['name'] == task.name: + logger.info('Terminating task: %s', task.name) + celery_app.control.revoke(worker_task['id'], terminate=True) + + # task name in both enable and disable, figure out if running or + # not and either stop or start + for name in confusables: + try: + service = Service.objects.get(service=name) + except Service.DoesNotExist: + logger.error('Unknown service: %s', name) + continue + + task = getattr(services_module, service.service) + is_active = False + for tasklist in inquisitor.active().values(): + for worker_task in tasklist: + if worker_task['name'] == task.name: + logger.info('Terminating task: %s', task.name) + is_active = True + celery_app.control.revoke(worker_task['id'], terminate=True) + if is_active: + # task not found in queue, wasn't running, activate + logger.info('Starting service query %s', name) + task.delay() diff --git a/viewer/cset_upload.py b/viewer/cset_upload.py index 0ccf7555..cad958b1 100644 --- a/viewer/cset_upload.py +++ b/viewer/cset_upload.py @@ -7,20 +7,21 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple +import numpy as np from openpyxl.utils import get_column_letter os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fragalysis.settings") import django django.setup() -from django.conf import settings - -logger = logging.getLogger(__name__) +from django.conf import settings +from django.core.exceptions import MultipleObjectsReturned from django.core.files.base import ContentFile from django.core.files.storage import default_storage +from django.db.models import F, TextField, Value +from django.db.models.expressions import Func from rdkit import Chem -from rdkit.Chem import Crippen, Descriptors from viewer.models import ( Compound, @@ -34,7 +35,13 @@ TextScoreValues, User, ) -from viewer.utils import add_props_to_sdf_molecule, is_url, word_count +from viewer.utils import add_props_to_sdf_molecule, alphanumerator, is_url, word_count + +logger = logging.getLogger(__name__) + + +# maximum distance between corresponding atoms in poses +_DIST_LIMIT = 0.5 def dataType(a_str: str) -> str: @@ -132,6 +139,7 @@ def __init__( version, zfile, zfile_hashvals, + computed_set_name, ): self.user_id = user_id self.sdf_filename = sdf_filename @@ -141,6 +149,7 @@ def __init__( self.version = version self.zfile = zfile self.zfile_hashvals = zfile_hashvals + self.computed_set_name = computed_set_name def process_pdb(self, pdb_code, zfile, zfile_hashvals) -> str | None: for key in zfile_hashvals.keys(): @@ -254,41 +263,63 @@ def get_site_observation( return site_obvs - def create_mol(self, inchi, long_inchi=None, name=None) -> Compound: + def create_mol(self, inchi, target, name=None) -> Compound: # check for an existing compound, returning a Compound - if long_inchi: - cpd = Compound.objects.filter(long_inchi=long_inchi) - sanitized_mol = Chem.MolFromInchi(long_inchi, sanitize=True) - else: - cpd = Compound.objects.filter(inchi=inchi) - sanitized_mol = Chem.MolFromInchi(inchi, sanitize=True) - - new_mol = cpd[0] if len(cpd) != 0 else Compound() - new_mol.smiles = Chem.MolToSmiles(sanitized_mol) - new_mol.inchi = inchi - if long_inchi: - new_mol.long_inchi = long_inchi - new_mol.identifier = name - - # descriptors - new_mol.mol_log_p = Crippen.MolLogP(sanitized_mol) - new_mol.mol_wt = float(Chem.rdMolDescriptors.CalcExactMolWt(sanitized_mol)) - new_mol.heavy_atom_count = Chem.Lipinski.HeavyAtomCount(sanitized_mol) - new_mol.heavy_atom_mol_wt = float(Descriptors.HeavyAtomMolWt(sanitized_mol)) - new_mol.nhoh_count = Chem.Lipinski.NHOHCount(sanitized_mol) - new_mol.no_count = Chem.Lipinski.NOCount(sanitized_mol) - new_mol.num_h_acceptors = Chem.Lipinski.NumHAcceptors(sanitized_mol) - new_mol.num_h_donors = Chem.Lipinski.NumHDonors(sanitized_mol) - new_mol.num_het_atoms = Chem.Lipinski.NumHeteroatoms(sanitized_mol) - new_mol.num_rot_bonds = Chem.Lipinski.NumRotatableBonds(sanitized_mol) - new_mol.num_val_electrons = Descriptors.NumValenceElectrons(sanitized_mol) - new_mol.ring_count = Chem.Lipinski.RingCount(sanitized_mol) - new_mol.tpsa = Chem.rdMolDescriptors.CalcTPSA(sanitized_mol) - - # make sure there is an id so inspirations can be added - new_mol.save() - - return new_mol + + sanitized_mol = Chem.MolFromInchi(inchi, sanitize=True) + Chem.RemoveStereochemistry(sanitized_mol) + inchi = Chem.inchi.MolToInchi(sanitized_mol) + inchi_key = Chem.InchiToInchiKey(inchi) + + try: + # NB! Max said there could be thousands of compounds per + # target so this distinct() here may become a problem + + # fmt: off + cpd = Compound.objects.filter( + computedmolecule__computed_set__target=target, + ).distinct().get( + inchi_key=inchi_key, + ) + # fmt: on + except Compound.DoesNotExist: + cpd = Compound( + smiles=Chem.MolToSmiles(sanitized_mol), + inchi=inchi, + inchi_key=inchi_key, + current_identifier=name, + ) + # This is a new compound. + cpd.save() + # This is a new compound. + # We must now set relationships to the Proposal that it applies to. + # We do this by copying the relationships from the Target. + num_target_proposals = len(target.project_id.all()) + assert num_target_proposals > 0 + if num_target_proposals > 1: + logger.warning( + 'Compound Target %s has more than one Proposal (%d)', + target.title, + num_target_proposals, + ) + for project in target.project_id.all(): + cpd.project_id.add(project) + except MultipleObjectsReturned as exc: + # NB! when processing new uploads, Compound is always + # fetched by inchi_key, so this shouldn't ever create + # duplicates. Ands LHS uploads do not create inchi_keys, + # so under normal operations duplicates should never + # occur. However there's nothing in the db to prevent + # this, so adding a catch clause and writing a meaningful + # message + logger.error( + 'Duplicate compounds for target %s with inchi key %s.', + target.title, + inchi_key, + ) + raise MultipleObjectsReturned from exc + + return cpd def set_props(self, cpd, props, compound_set) -> List[ScoreDescription]: if 'ref_mols' and 'ref_pdb' not in list(props.keys()): @@ -322,13 +353,9 @@ def set_mol( smiles = Chem.MolToSmiles(mol) inchi = Chem.inchi.MolToInchi(mol) molecule_name = mol.GetProp('_Name') - long_inchi = None - if len(inchi) > 255: - long_inchi = inchi - inchi = inchi[:254] compound: Compound = self.create_mol( - inchi, name=molecule_name, long_inchi=long_inchi + inchi, compound_set.target, name=molecule_name ) insp = mol.GetProp('ref_mols') @@ -353,12 +380,7 @@ def set_mol( 'No matching molecules found for inspiration frag ' + i ) - if qs.count() > 1: - ids = [m.cmpd.id for m in qs] - ind = ids.index(max(ids)) - ref = qs[ind] - elif qs.count() == 1: - ref = qs[0] + ref = qs.order_by('-cmpd_id').first() insp_frags.append(ref) @@ -385,14 +407,75 @@ def set_mol( # Need a ComputedMolecule before saving. # Check if anything exists already... - existing_computed_molecules = ComputedMolecule.objects.filter( - molecule_name=molecule_name, smiles=smiles, computed_set=compound_set + + # I think, realistically, I only need to check compound + # fmt: off + qs = ComputedMolecule.objects.filter( + compound=compound, + ).annotate( + # names come in format: + # target_name-sequential number-sequential letter, + # e.g. A71EV2A-1-a, hence grabbing the 3rd column + suffix=Func( + F('name'), + Value('-'), + Value(3), + function='split_part', + output_field=TextField(), + ), ) - computed_molecule: Optional[ComputedMolecule] = None + if qs.exists(): + suffix = next( + alphanumerator(start_from=qs.order_by('-suffix').first().suffix) + ) + else: + suffix = 'a' + + # distinct is ran on indexed field, so shouldn't be a problem + number = ComputedMolecule.objects.filter( + computed_set__target=compound_set.target, + ).values('id').distinct().count() + 1 + # fmt: on + + name = f'v{number}{suffix}' + + existing_computed_molecules = [] + for k in qs: + kmol = Chem.MolFromMolBlock(k.sdf_info) + if kmol: + # find distances between corresponding atoms of the + # two conformers. if any one exceeds the _DIST_LIMIT, + # consider it to be a new ComputedMolecule + try: + _, _, atom_map = Chem.rdMolAlign.GetBestAlignmentTransform( + mol, kmol + ) + except RuntimeError as exc: + msg = ( + f'Failed to find alignment between {k.molecule_name} ' + + f'and {mol.GetProp("original ID")}' + ) + logger.error(msg) + raise RuntimeError(msg) from exc + + molconf = mol.GetConformer() + kmolconf = kmol.GetConformer() + small_enough = True + for mol_atom, kmol_atom in atom_map: + molpos = np.array(molconf.GetAtomPosition(mol_atom)) + kmolpos = np.array(kmolconf.GetAtomPosition(kmol_atom)) + distance = np.linalg.norm(molpos - kmolpos) + if distance >= _DIST_LIMIT: + small_enough = False + break + if small_enough: + existing_computed_molecules.append(k) + if len(existing_computed_molecules) == 1: - logger.info( - 'Using existing ComputedMolecule %s', existing_computed_molecules[0] + logger.warning( + 'Using existing ComputedMolecule %s and overwriting its metadata', + existing_computed_molecules[0], ) computed_molecule = existing_computed_molecules[0] elif len(existing_computed_molecules) > 1: @@ -400,10 +483,10 @@ def set_mol( for exist in existing_computed_molecules: logger.info('Deleting ComputedMolecule %s', exist) exist.delete() - computed_molecule = ComputedMolecule() - if not computed_molecule: + computed_molecule = ComputedMolecule(name=name) + else: logger.info('Creating new ComputedMolecule') - computed_molecule = ComputedMolecule() + computed_molecule = ComputedMolecule(name=name) if isinstance(ref_so, SiteObservation): code = ref_so.code @@ -414,14 +497,15 @@ def set_mol( pdb_info = ref_so lhs_so = None - assert computed_molecule + # I don't quite understand why the overwrite of existing + # compmol.. but this is how it was, not touching it now + # update: I think it's about updating metadata. moving + # name attribute out so it won't get overwritten computed_molecule.compound = compound - computed_molecule.computed_set = compound_set computed_molecule.sdf_info = Chem.MolToMolBlock(mol) computed_molecule.site_observation_code = code computed_molecule.reference_code = code computed_molecule.molecule_name = molecule_name - computed_molecule.name = f"{target}-{computed_molecule.identifier}" computed_molecule.smiles = smiles computed_molecule.pdb = lhs_so # TODO: this is wrong @@ -447,6 +531,8 @@ def set_mol( # Done computed_molecule.save() + compound_set.computed_molecules.add(computed_molecule) + # No update the molecule in the original file... add_props_to_sdf_molecule( sdf_file=filename, @@ -530,50 +616,51 @@ def task(self) -> ComputedSet: ) # Do we have any existing ComputedSets? - # Ones with the same method and upload date? - today: datetime.date = datetime.date.today() - existing_sets: List[ComputedSet] = ComputedSet.objects.filter( - method=truncated_submitter_method, upload_date=today - ).all() - # If so, find the one with the highest ordinal. - latest_ordinal: int = 0 - for exiting_set in existing_sets: - assert exiting_set.md_ordinal > 0 - if exiting_set.md_ordinal > latest_ordinal: - latest_ordinal = exiting_set.md_ordinal - if latest_ordinal: - logger.info( - 'Found existing ComputedSets for method "%s" on %s (%d) with ordinal=%d', - truncated_submitter_method, - str(today), - len(existing_sets), - latest_ordinal, + try: + computed_set = ComputedSet.objects.get(name=self.computed_set_name) + # refresh some attributes + computed_set.md_ordinal = F('md_ordinal') + 1 + computed_set.upload_date = datetime.date.today() + computed_set.save() + except ComputedSet.DoesNotExist: + # no, create new + + today: datetime.date = datetime.date.today() + new_ordinal: int = 1 + try: + target = Target.objects.get(title=self.target) + except Target.DoesNotExist as exc: + # probably wrong target name supplied + logger.error('Target %s does not exist', self.target) + raise Target.DoesNotExist from exc + + cs_name: str = ( + f'{truncated_submitter_method}-{str(today)}-' + + f'{get_column_letter(new_ordinal)}' ) - # ordinals are 1-based - new_ordinal: int = latest_ordinal + 1 - - # The computed set "name" consists of the "method", - # today's date and a 2-digit ordinal. The ordinal - # is used to distinguish between computed sets uploaded - # with the same method on the same day. - assert new_ordinal > 0 - cs_name: str = f"{truncated_submitter_method}-{str(today)}-{get_column_letter(new_ordinal)}" - logger.info('Creating new ComputedSet "%s"', cs_name) - - computed_set: ComputedSet = ComputedSet() - computed_set.name = cs_name - computed_set.md_ordinal = new_ordinal - computed_set.upload_date = today - computed_set.method = self.submitter_method[: ComputedSet.LENGTH_METHOD] - computed_set.target = Target.objects.get(title=self.target) - computed_set.spec_version = float(self.version.strip('ver_')) - if self.user_id: - computed_set.owner_user = User.objects.get(id=self.user_id) - else: - # The User ID may only be None if AUTHENTICATE_UPLOAD is False. - # Here the ComputedSet owner will take on a default (anonymous) value. - assert settings.AUTHENTICATE_UPLOAD is False - computed_set.save() + logger.info('Creating new ComputedSet "%s"', cs_name) + + computed_set = ComputedSet( + name=cs_name, + md_ordinal=new_ordinal, + upload_date=today, + method=self.submitter_method[: ComputedSet.LENGTH_METHOD], + target=target, + spec_version=float(self.version.strip('ver_')), + ) + if self.user_id: + try: + computed_set.owner_user = User.objects.get(id=self.user_id) + except User.DoesNotExist as exc: + logger.error('User %s does not exist', self.user_id) + raise User.DoesNotExist from exc + + else: + # The User ID may only be None if AUTHENTICATE_UPLOAD is False. + # Here the ComputedSet owner will take on a default (anonymous) value. + assert settings.AUTHENTICATE_UPLOAD is False + + computed_set.save() # check compound set folder exists. cmp_set_folder = os.path.join( diff --git a/viewer/download_structures.py b/viewer/download_structures.py index 0a25850c..c8a2778f 100644 --- a/viewer/download_structures.py +++ b/viewer/download_structures.py @@ -104,6 +104,25 @@ def __init__(self, category): # fmt: on +class UploadTagSubquery(Subquery): + """Annotate SiteObservation with tag of given category""" + + def __init__(self, category): + # fmt: off + query = SiteObservationTag.objects.filter( + pk=Subquery( + SiteObvsSiteObservationTag.objects.filter( + site_observation=OuterRef(OuterRef('pk')), + site_obvs_tag__category=TagCategory.objects.get( + category=category, + ), + ).values('site_obvs_tag')[:1] + ) + ).values('upload_name')[0:1] + super().__init__(query) + # fmt: on + + class CuratedTagSubquery(Exists): """Annotate SiteObservation with tag of given category""" @@ -140,6 +159,10 @@ class ArchiveFile: 'ligand_pdb': {}, 'ligand_mol': {}, 'ligand_smiles': {}, + # additional ccp4 files, issue 1448 + 'event_file_crystallographic': {}, + 'diff_file_crystallographic': {}, + 'sigmaa_file_crystallographic': {}, }, 'molecules': { 'sdf_files': {}, @@ -215,29 +238,34 @@ def _read_and_patch_molecule_name(path, molecule_name=None): return content -def _patch_molecule_name(site_observation): - """Patch the MOL or SDF file with molecule name. +# def _patch_molecule_name(site_observation): +# """Patch the MOL or SDF file with molecule name. - Processes the content of ligand_mol attribute of the - site_observation object. Returns the content as string. +# Processes the content of ligand_mol attribute of the +# site_observation object. Returns the content as string. - Alternative to _read_and_patch_molecule_name function above - which operates on files. As ligand_mol is now stored as text, - slightly different approach was necessary. +# Alternative to _read_and_patch_molecule_name function above +# which operates on files. As ligand_mol is now stored as text, +# slightly different approach was necessary. - """ - logger.debug('Patching MOL/SDF of "%s"', site_observation) +# """ +# logger.debug('Patching MOL/SDF of "%s"', site_observation) - # Now read the file, checking the first line - # and setting it to the molecule name if it's blank. - lines = site_observation.ligand_mol_file.split('\n') - if not lines[0].strip(): - lines[0] = site_observation.long_code +# path = Path(settings.MEDIA_ROOT).joinpath(site_observation.ligand_mol.name) +# with contextlib.suppress(TypeError, FileNotFoundError): +# with open(path, "r", encoding="utf-8") as f: +# lines = f.readlines() + +# # Now read the file, checking the first line +# # and setting it to the molecule name if it's blank. +# # lines = site_observation.ligand_mol_file.split('\n') +# if not lines[0].strip(): +# lines[0] = site_observation.long_code - # the db contents is mol file but what's requested here is - # sdf. add sdf separator - lines.append('$$$$\n') - return '\n'.join(lines) +# # the db contents is mol file but what's requested here is +# # sdf. add sdf separator +# lines.append('$$$$\n') +# return '\n'.join(lines) def _add_file_to_zip_aligned(ziparchive, code, archive_file): @@ -276,7 +304,7 @@ def _add_file_to_zip_aligned(ziparchive, code, archive_file): elif archive_file.site_observation: ziparchive.writestr( archive_file.archive_path, - _patch_molecule_name(archive_file.site_observation), + _read_and_patch_molecule_name(filepath, archive_file.site_observation), ) return True else: @@ -303,7 +331,9 @@ def _add_file_to_sdf(combined_sdf_file, archive_file): if archive_file.path and archive_file.path != 'None': with open(combined_sdf_file, 'a', encoding='utf-8') as f_out: - patched_sdf_content = _patch_molecule_name(archive_file.site_observation) + patched_sdf_content = _read_and_patch_molecule_name( + archive_file.path, archive_file.site_observation + ) f_out.write(patched_sdf_content) return True else: @@ -423,14 +453,34 @@ def _metadata_file_zip(ziparchive, target, site_observations): logger.info('+ Processing metadata') annotations = {} - values = ['code', 'longcode', 'cmpd__compound_code', 'smiles', 'downloaded'] - header = ['Code', 'Long code', 'Compound code', 'Smiles', 'Downloaded'] + values = [ + 'code', + 'longcode', + 'experiment__code', + 'cmpd__compound_code', + 'smiles', + 'canon_site_conf__canon_site__centroid_res', + 'downloaded', + ] + header = [ + 'Code', + 'Long code', + 'Experiment code', + 'Compound code', + 'Smiles', + 'Centroid res', + 'Downloaded', + ] for category in TagCategory.objects.filter(category__in=TAG_CATEGORIES): tag = f'tag_{category.category.lower()}' + upload_tag = f'upload_tag_{category.category.lower()}' values.append(tag) - header.append(category.category) + header.append(f'{category.category} alias') annotations[tag] = TagSubquery(category.category) + values.append(upload_tag) + header.append(f'{category.category} upload name') + annotations[upload_tag] = UploadTagSubquery(category.category) pattern = re.compile(r'\W+') # non-alphanumeric characters for tag in SiteObservationTag.objects.filter( @@ -485,39 +535,36 @@ def _extra_files_zip(ziparchive, target): num_processed = 0 num_extra_dir = 0 - for experiment_upload in target.experimentupload_set.order_by('commit_datetime'): - extra_files = ( - Path(settings.MEDIA_ROOT) - .joinpath(settings.TARGET_LOADER_MEDIA_DIRECTORY) - .joinpath(experiment_upload.task_id) - ) + # taking the latest upload for now - # taking the latest upload for now - # add unpacked zip directory - extra_files = [d for d in list(extra_files.glob("*")) if d.is_dir()][0] - - # add upload_[d] dir - extra_files = next(extra_files.glob("upload_*")) - extra_files = extra_files.joinpath('extra_files') - - logger.debug('extra_files path 2: %s', extra_files) - logger.info('Processing extra files (%s)...', extra_files) - - if extra_files.is_dir(): - num_extra_dir = num_extra_dir + 1 - for dirpath, _, files in os.walk(extra_files): - for file in files: - filepath = os.path.join(dirpath, file) - logger.info('Adding extra file "%s"...', filepath) - ziparchive.write( - filepath, - os.path.join( - f'{_ZIP_FILEPATHS["extra_files"]}_{num_extra_dir}', file - ), - ) - num_processed += 1 - else: - logger.info('Directory does not exist (%s)...', extra_files) + experiment_upload = target.experimentupload_set.order_by('commit_datetime').last() + extra_files = ( + Path(settings.MEDIA_ROOT) + .joinpath(settings.TARGET_LOADER_MEDIA_DIRECTORY) + .joinpath(target.zip_archive.name) + .joinpath(experiment_upload.upload_data_dir) + ) + + extra_files = extra_files.joinpath('extra_files') + + logger.debug('extra_files path 2: %s', extra_files) + logger.info('Processing extra files (%s)...', extra_files) + + if extra_files.is_dir(): + num_extra_dir = num_extra_dir + 1 + for dirpath, _, files in os.walk(extra_files): + for file in files: + filepath = os.path.join(dirpath, file) + logger.info('Adding extra file "%s"...', filepath) + ziparchive.write( + filepath, + os.path.join( + f'{_ZIP_FILEPATHS["extra_files"]}_{num_extra_dir}', file + ), + ) + num_processed += 1 + else: + logger.info('Directory does not exist (%s)...', extra_files) if num_processed == 0: logger.info('No extra files found') @@ -528,27 +575,22 @@ def _extra_files_zip(ziparchive, target): def _yaml_files_zip(ziparchive, target, transforms_requested: bool = False) -> None: """Add all yaml files (except transforms) from upload to ziparchive""" - for experiment_upload in target.experimentupload_set.order_by('commit_datetime'): + for experiment_upload in target.experimentupload_set.all(): yaml_paths = ( Path(settings.MEDIA_ROOT) .joinpath(settings.TARGET_LOADER_MEDIA_DIRECTORY) - .joinpath(experiment_upload.task_id) + .joinpath(target.zip_archive.name) + .joinpath(experiment_upload.upload_data_dir) ) transforms = [ Path(f.name).name for f in ( + experiment_upload.conformer_site_transforms, experiment_upload.neighbourhood_transforms, - experiment_upload.neighbourhood_transforms, - experiment_upload.neighbourhood_transforms, + experiment_upload.reference_structure_transforms, ) ] - # taking the latest upload for now - # add unpacked zip directory - yaml_paths = [d for d in list(yaml_paths.glob("*")) if d.is_dir()][0] - - # add upload_[d] dir - yaml_paths = next(yaml_paths.glob("upload_*")) archive_path = Path('yaml_files').joinpath(yaml_paths.parts[-1]) @@ -742,6 +784,25 @@ def _create_structures_dict(site_obvs, protein_params, other_params): # Read through zip_params to compile the parameters zip_contents: Dict[str, Any] = copy.deepcopy(zip_template) + site_obvs = site_obvs.annotate( + # would there be any point in + # a) adding a method to SiteObservation model_attr + # b) adding the value to database directly? + longlongcode=Concat( + F('experiment__code'), + Value('_'), + F('chain_id'), + Value('_'), + F('seq_id'), + Value('_'), + F('version'), + Value('_'), + F('canon_site_conf__canon_site__name'), + Value('+'), + F('canon_site_conf__canon_site__version'), + output_field=CharField(), + ), + ) for so in site_obvs: for param in protein_params: if protein_params[param] is True: @@ -764,9 +825,12 @@ def _create_structures_dict(site_obvs, protein_params, other_params): for f in model_attr: # here the model_attr is already stringified try: - exp_path = re.search(r"x\d*", so.code).group(0) # type: ignore[union-attr] - except AttributeError: - logger.error('Unexpected shortcodeformat: %s', so.code) + exp_path = so.experiment.code.split('-x')[1] + except IndexError: + logger.error( + 'Unexpected experiment code format: %s', + so.experiment.code, + ) exp_path = so.code apath = Path('crystallographic_files').joinpath(exp_path) @@ -808,10 +872,11 @@ def _create_structures_dict(site_obvs, protein_params, other_params): apath.joinpath( Path(model_attr.name) .parts[-1] - .replace(so.longcode, so.code) + .replace(so.longlongcode, so.code) ) ) else: + # file not in upload archive_path = str(apath.joinpath(param)) afile = [ @@ -820,31 +885,58 @@ def _create_structures_dict(site_obvs, protein_params, other_params): archive_path=archive_path, ) ] + else: logger.warning('Unexpected param: %s', param) continue zip_contents['proteins'][param][so.code] = afile + # add additional ccp4 files (issue 1448) + ccps = ('sigmaa_file', 'diff_file', 'event_file') + if param in ccps: + # these only come from siteobservation object + model_attr = getattr(so, param) + if model_attr and model_attr != 'None': + apath = Path('aligned_files').joinpath(so.code) + ccp_path = Path(model_attr.name) + path = ccp_path.parent.joinpath( + f'{ccp_path.stem}_crystallographic{ccp_path.suffix}' + ) + archive_path = str( + apath.joinpath( + path.parts[-1].replace(so.longlongcode, so.code) + ) + ) + + afile = [ + ArchiveFile( + path=str(path), + archive_path=archive_path, + ) + ] + zip_contents['proteins'][f'{param}_crystallographic'][ + so.code + ] = afile + zip_contents['molecules']['single_sdf_file'] = other_params['single_sdf_file'] zip_contents['molecules']['sdf_info'] = other_params['sdf_info'] - # sdf information is held as a file on the Molecule record. if other_params['sdf_info'] or other_params['single_sdf_file']: num_molecules_collected = 0 num_missing_sd_files = 0 for so in site_obvs: - if so.ligand_mol_file: + if so.ligand_mol: # There is an SD file (normal) - # sdf info is now kept as text in db field archive_path = str( Path('aligned_files').joinpath(so.code).joinpath(f'{so.code}.sdf') ) + file_path = str(Path(settings.MEDIA_ROOT).joinpath(so.ligand_mol.name)) # path is ignored when writing sdfs but mandatory field zip_contents['molecules']['sdf_files'].update( { ArchiveFile( - path=archive_path, + path=file_path, archive_path=archive_path, site_observation=so, ): so.code @@ -854,7 +946,7 @@ def _create_structures_dict(site_obvs, protein_params, other_params): else: # No file value (odd). logger.warning( - "SiteObservation record's 'ligand_mol_file' isn't set (%s)", so + "SiteObservation record's 'ligand_mol' isn't set (%s)", so ) num_missing_sd_files += 1 diff --git a/viewer/filters.py b/viewer/filters.py index d5e62e03..d0df524a 100644 --- a/viewer/filters.py +++ b/viewer/filters.py @@ -1,3 +1,5 @@ +import logging + import django_filters from django_filters import rest_framework as filters @@ -12,6 +14,8 @@ XtalformSite, ) +logger = logging.getLogger(__name__) + class SnapshotFilter(filters.FilterSet): session_project = django_filters.CharFilter( diff --git a/viewer/fixtures/tagcategories.json b/viewer/fixtures/tagcategories.json index e502eb50..e9155f3f 100644 --- a/viewer/fixtures/tagcategories.json +++ b/viewer/fixtures/tagcategories.json @@ -22,7 +22,7 @@ "pk": 3, "fields": { "category": "CrystalformSites", - "colour": "0099ff", + "colour": "ff9900", "description": null } }, diff --git a/viewer/management/commands/README.md b/viewer/management/commands/README.md index ff00d8d6..97e461f3 100644 --- a/viewer/management/commands/README.md +++ b/viewer/management/commands/README.md @@ -45,3 +45,32 @@ commit; centre of mass was used as an equivalent grouping. If there is no sites.csv file, the function will look for molgroups with a description of 'c_of_m' and create site tags for those instead with a name 'c_of_m_'. + + +## start_service_queries + +Defined in app `service_status`. + +``` +python manage.py start_service_queries +``` + +Starts the external service health check tasks. Reads the required +tasks from `ENABLE_SERVICE_STATUS` environment variable and launches +the corresponding task (test function must be defined in +`service_status/services.py`). + + +## services + +Defined in app `service_status`. + +``` +python manage.py services --enable [comma separated service names] + +# or + +python manage.py services --disable [comma separated service names] +``` + +Starts or stops service health check queries individually. diff --git a/viewer/management/commands/curated_tags.py b/viewer/management/commands/curated_tags.py new file mode 100644 index 00000000..0956d66b --- /dev/null +++ b/viewer/management/commands/curated_tags.py @@ -0,0 +1,27 @@ +from django.core.management.base import BaseCommand + +from viewer.utils import dump_curated_tags, restore_curated_tags + + +class Command(BaseCommand): + help = "Dump or load curated tags" + + def add_arguments(self, parser): + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--dump", metavar="", type=str, help="Save to file" + ) + group.add_argument( + "--load", + metavar="JSON file>", + type=str, + help="Load file", + ) + + def handle(self, *args, **kwargs): + # Unused args + del args + if kwargs["dump"]: + dump_curated_tags(filename=kwargs["dump"]) + if kwargs["load"]: + restore_curated_tags(filename=kwargs["load"]) diff --git a/viewer/managers.py b/viewer/managers.py index 12d9937d..29985fe3 100644 --- a/viewer/managers.py +++ b/viewer/managers.py @@ -315,3 +315,101 @@ def filter_qs(self): def by_target(self, target): return self.get_queryset().filter_qs().filter(target=target.id) + + +class SnapshotQueryset(QuerySet): + def filter_qs(self): + Snapshot = apps.get_model("viewer", "Snapshot") + qs = Snapshot.objects.prefetch_related( + "session_project", + "session_project__target", + ).annotate( + target=F("session_project__target"), + ) + + return qs + + +class SnapshotDataManager(Manager): + def get_queryset(self): + return SnapshotQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class SnapshotActionsQueryset(QuerySet): + def filter_qs(self): + SnapshotActions = apps.get_model("viewer", "SnapshotActions") + qs = SnapshotActions.objects.prefetch_related( + "session_project", + "session_project__target", + ).annotate( + target=F("session_project__target"), + ) + + return qs + + +class SnapshotActionsDataManager(Manager): + def get_queryset(self): + return SnapshotActionsQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class SessionActionsQueryset(QuerySet): + def filter_qs(self): + SessionActions = apps.get_model("viewer", "SessionActions") + qs = SessionActions.objects.prefetch_related( + "session_project", + "session_project__target", + ).annotate( + target=F("session_project__target"), + ) + + return qs + + +class SessionActionsDataManager(Manager): + def get_queryset(self): + return SessionActionsQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class CompoundIdentifierQueryset(QuerySet): + def filter_qs(self): + CompoundIdentifier = apps.get_model("viewer", "CompoundIdentifier") + qs = CompoundIdentifier.objects.prefetch_related( + "cmpd_id__siteobservation", + "cmpd_id__siteobservation__experiment", + "cmpd_id__siteobservation__experiment__experiment_upload", + "cmpd_id__siteobservation__experiment__experiment_upload__target", + ).annotate( + target=F("cmpd_id__siteobservation__experiment__experiment_upload__target"), + ) + + return qs + + +class CompoundIdentifierDataManager(Manager): + def get_queryset(self): + return CompoundIdentifierQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) diff --git a/viewer/migrations/0056_compound_inchi_key.py b/viewer/migrations/0056_compound_inchi_key.py new file mode 100644 index 00000000..d0591072 --- /dev/null +++ b/viewer/migrations/0056_compound_inchi_key.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-06-11 13:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0055_merge_20240516_1003'), + ] + + operations = [ + migrations.AddField( + model_name='compound', + name='inchi_key', + field=models.CharField(blank=True, db_index=True, max_length=80), + ), + ] diff --git a/viewer/migrations/0057_auto_20240612_1348.py b/viewer/migrations/0057_auto_20240612_1348.py new file mode 100644 index 00000000..1dd8e5ac --- /dev/null +++ b/viewer/migrations/0057_auto_20240612_1348.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.25 on 2024-06-12 13:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0056_compound_inchi_key'), + ] + + operations = [ + migrations.AddField( + model_name='computedmolecule', + name='version', + field=models.PositiveSmallIntegerField(default=1), + ), + migrations.AlterField( + model_name='compound', + name='inchi_key', + field=models.CharField(blank=True, db_index=True, max_length=27), + ), + ] diff --git a/viewer/migrations/0058_auto_20240614_1016.py b/viewer/migrations/0058_auto_20240614_1016.py new file mode 100644 index 00000000..8e296ed6 --- /dev/null +++ b/viewer/migrations/0058_auto_20240614_1016.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2.25 on 2024-06-14 10:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0057_auto_20240612_1348'), + ] + + operations = [ + migrations.CreateModel( + name='ComputedSetComputedMolecule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('computed_molecule', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='viewer.computedmolecule')), + ('computed_set', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='viewer.computedset')), + ], + ), + migrations.AddConstraint( + model_name='computedsetcomputedmolecule', + constraint=models.UniqueConstraint(fields=('computed_set', 'computed_molecule'), name='unique_computedsetcomputedmolecule'), + ), + migrations.RemoveField( + model_name='computedmolecule', + name='computed_set', + ), + migrations.AddField( + model_name='computedset', + name='computed_molecules', + field=models.ManyToManyField(related_name='computed_set', through='viewer.ComputedSetComputedMolecule', to='viewer.ComputedMolecule'), + ), + ] diff --git a/viewer/migrations/0059_remove_computedmolecule_version.py b/viewer/migrations/0059_remove_computedmolecule_version.py new file mode 100644 index 00000000..c6baece9 --- /dev/null +++ b/viewer/migrations/0059_remove_computedmolecule_version.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.25 on 2024-07-10 08:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0058_auto_20240614_1016'), + ] + + operations = [ + migrations.RemoveField( + model_name='computedmolecule', + name='version', + ), + ] diff --git a/viewer/migrations/0060_canonsite_centroid_res.py b/viewer/migrations/0060_canonsite_centroid_res.py new file mode 100644 index 00000000..97150507 --- /dev/null +++ b/viewer/migrations/0060_canonsite_centroid_res.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-07-29 12:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0059_remove_computedmolecule_version'), + ] + + operations = [ + migrations.AddField( + model_name='canonsite', + name='centroid_res', + field=models.TextField(null=True), + ), + ] diff --git a/viewer/migrations/0061_auto_20240905_0756.py b/viewer/migrations/0061_auto_20240905_0756.py new file mode 100644 index 00000000..d6329a03 --- /dev/null +++ b/viewer/migrations/0061_auto_20240905_0756.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.25 on 2024-09-05 07:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0060_canonsite_centroid_res'), + ] + + operations = [ + migrations.RemoveField( + model_name='historicalsiteobservation', + name='ligand_mol_file', + ), + migrations.RemoveField( + model_name='siteobservation', + name='ligand_mol_file', + ), + ] diff --git a/viewer/migrations/0061_auto_20240905_1500.py b/viewer/migrations/0061_auto_20240905_1500.py new file mode 100644 index 00000000..91ad093c --- /dev/null +++ b/viewer/migrations/0061_auto_20240905_1500.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.25 on 2024-09-05 15:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0060_canonsite_centroid_res'), + ] + + operations = [ + migrations.AddField( + model_name='sessionprojecttag', + name='short_tag', + field=models.TextField(help_text='The generated shorter version of tag (without target name)', null=True), + ), + migrations.AddField( + model_name='siteobservationtag', + name='short_tag', + field=models.TextField(help_text='The generated shorter version of tag (without target name)', null=True), + ), + migrations.AlterField( + model_name='sessionprojecttag', + name='upload_name', + field=models.CharField(help_text='The generated long name of the tag', max_length=200), + ), + migrations.AlterField( + model_name='siteobservationtag', + name='upload_name', + field=models.CharField(help_text='The generated long name of the tag', max_length=200), + ), + ] diff --git a/viewer/migrations/0062_experiment_code_prefix.py b/viewer/migrations/0062_experiment_code_prefix.py new file mode 100644 index 00000000..2a585e92 --- /dev/null +++ b/viewer/migrations/0062_experiment_code_prefix.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-09-05 15:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0061_auto_20240905_1500'), + ] + + operations = [ + migrations.AddField( + model_name='experiment', + name='code_prefix', + field=models.TextField(null=True), + ), + ] diff --git a/viewer/migrations/0063_merge_20240906_1243.py b/viewer/migrations/0063_merge_20240906_1243.py new file mode 100644 index 00000000..dad50214 --- /dev/null +++ b/viewer/migrations/0063_merge_20240906_1243.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.25 on 2024-09-06 12:43 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0061_auto_20240905_0756'), + ('viewer', '0062_experiment_code_prefix'), + ] + + operations = [ + ] diff --git a/viewer/models.py b/viewer/models.py index 0d38914f..f17b6a06 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -17,10 +17,14 @@ CanonSiteConfDataManager, CanonSiteDataManager, CompoundDataManager, + CompoundIdentifierDataManager, ExperimentDataManager, PoseDataManager, QuatAssemblyDataManager, + SessionActionsDataManager, SiteObservationDataManager, + SnapshotActionsDataManager, + SnapshotDataManager, XtalformDataManager, XtalformQuatAssemblyDataManager, XtalformSiteDataManager, @@ -123,7 +127,12 @@ def __str__(self) -> str: return f"{self.title}" def __repr__(self) -> str: - return "" % (self.id, self.title, self.project_id) + return "" % ( + self.id, + self.title, + self.display_name, + self.project_id, + ) class ExperimentUpload(models.Model): @@ -186,11 +195,18 @@ def get_upload_path(self): return ( Path(settings.MEDIA_ROOT) .joinpath(settings.TARGET_LOADER_MEDIA_DIRECTORY) - .joinpath(self.task_id) - .joinpath(Path(str(self.file)).stem) + .joinpath(self.target.zip_archive.name) .joinpath(self.upload_data_dir) ) + def get_download_path(self): + """The path to the original uploaded file, used during downloads""" + return ( + Path(settings.MEDIA_ROOT) + .joinpath(settings.TARGET_LOADER_MEDIA_DIRECTORY) + .joinpath(Path(str(self.file))) + ) + class Experiment(models.Model): experiment_upload = models.ForeignKey(ExperimentUpload, on_delete=models.CASCADE) @@ -209,6 +225,7 @@ class Experiment(models.Model): type = models.PositiveSmallIntegerField(null=True) pdb_sha256 = models.TextField(null=True) prefix_tooltip = models.TextField(null=True) + code_prefix = models.TextField(null=True) compounds = models.ManyToManyField( "Compound", through="ExperimentCompound", @@ -256,6 +273,7 @@ class Compound(models.Model): ) description = models.TextField(blank=True, null=True) comments = models.TextField(blank=True, null=True) + inchi_key = models.CharField(db_index=True, max_length=27, blank=True) objects = models.Manager() filter_manager = CompoundDataManager() @@ -390,6 +408,7 @@ class CanonSite(Versionable, models.Model): canon_site_num = models.IntegerField( null=True, help_text="numeric canon site id (enumerated on creation)" ) + centroid_res = models.TextField(null=True) objects = models.Manager() filter_manager = CanonSiteDataManager() @@ -515,7 +534,6 @@ class SiteObservation(Versionable, models.Model): smiles = models.TextField() seq_id = models.IntegerField() chain_id = models.CharField(max_length=1) - ligand_mol_file = models.TextField(null=True) ligand_mol = models.FileField( upload_to="target_loader_data/", null=True, max_length=255 ) @@ -561,6 +579,9 @@ class CompoundIdentifier(models.Model): url = models.URLField(max_length=URL_LENGTH, null=True) name = models.CharField(max_length=NAME_LENGTH) + objects = models.Manager() + filter_manager = CompoundIdentifierDataManager() + def __str__(self) -> str: return f"{self.name}" @@ -676,6 +697,9 @@ class SessionActions(models.Model): last_update_date = models.DateTimeField(default=timezone.now) actions = models.JSONField(encoder=DjangoJSONEncoder) + objects = models.Manager() + filter_manager = SessionActionsDataManager() + def __str__(self) -> str: return f"{self.author}" @@ -731,6 +755,9 @@ class Snapshot(models.Model): help_text='Optional JSON field containing name/value pairs for future use', ) + objects = models.Manager() + filter_manager = SnapshotDataManager() + def __str__(self) -> str: return f"{self.title}" @@ -767,6 +794,9 @@ class SnapshotActions(models.Model): last_update_date = models.DateTimeField(default=timezone.now) actions = models.JSONField(encoder=DjangoJSONEncoder) + objects = models.Manager() + filter_manager = SnapshotActionsDataManager() + def __str__(self) -> str: return f"{self.author}" @@ -946,6 +976,12 @@ class ComputedSet(models.Model): upload_datetime = models.DateTimeField( null=True, help_text="The datetime the upload was completed" ) + computed_molecules = models.ManyToManyField( + "ComputedMolecule", + through="ComputedSetComputedMolecule", + through_fields=("computed_set", "computed_molecule"), + related_name="computed_set", + ) def __str__(self) -> str: target_title: str = self.target.title if self.target else "None" @@ -973,7 +1009,6 @@ class ComputedMolecule(models.Model): null=True, blank=True, ) - computed_set = models.ForeignKey(ComputedSet, on_delete=models.CASCADE) name = models.CharField( max_length=50, help_text="A combination of Target and Identifier" ) @@ -1030,6 +1065,24 @@ def __repr__(self) -> str: ) +class ComputedSetComputedMolecule(models.Model): + computed_set = models.ForeignKey(ComputedSet, null=False, on_delete=models.CASCADE) + computed_molecule = models.ForeignKey( + ComputedMolecule, null=False, on_delete=models.CASCADE + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=[ + "computed_set", + "computed_molecule", + ], + name="unique_computedsetcomputedmolecule", + ), + ] + + class ScoreDescription(models.Model): """The names and descriptions of scores that the user uploads with each computed set molecule.""" @@ -1237,11 +1290,15 @@ class Meta: class Tag(models.Model): tag = models.CharField(max_length=200, help_text="The (unique) name of the tag") + short_tag = models.TextField( + null=True, + help_text="The generated shorter version of tag (without target name)", + ) tag_prefix = models.TextField( null=True, help_text="Tag prefix for auto-generated tags" ) upload_name = models.CharField( - max_length=200, help_text="The generated name of the tag" + max_length=200, help_text="The generated long name of the tag" ) category = models.ForeignKey(TagCategory, on_delete=models.CASCADE) target = models.ForeignKey(Target, on_delete=models.CASCADE) diff --git a/viewer/permissions.py b/viewer/permissions.py new file mode 100644 index 00000000..b9541013 --- /dev/null +++ b/viewer/permissions.py @@ -0,0 +1,76 @@ +import logging + +from rest_framework import permissions +from rest_framework.exceptions import PermissionDenied + +from api.security import ISPyBSafeQuerySet + +_ISPYB_SAFE_QUERY_SET = ISPyBSafeQuerySet() + +logger = logging.getLogger(__name__) + + +class IsObjectProposalMember(permissions.BasePermission): + """ + Custom permissions to only allow write-access to objects (changes) by users + who are members of the object's proposals. This permissions class should be used + in any view that needs to restrict object modifications to users who are members of + at least one of the object's proposals. This class can be used for objects that + either have one proposal or many. + + If the object has no proposals, the user is granted access. + """ + + def has_object_permission(self, request, view, obj): + # Here we check that the user has access to any proposal the object belongs to. + # Firstly, users must be authenticated + if not request.user.is_authenticated: + return False + # Protect ourselves from views that do not (oddly) + # have a property called 'filter_permissions'... + if not hasattr(view, "filter_permissions"): + raise AttributeError( + "The view object must define a 'filter_permissions' property" + ) + # The object's proposal records (one or many) can be obtained via + # the view's 'filter_permissions' property. A standard + # django property reference, e.g. 'target__project_id'. + object_proposals = [] + attr_value = getattr(obj, view.filter_permissions) + + try: + attr_value = getattr(obj, view.filter_permissions) + except AttributeError as exc: + # Something's gone wrong trying to lookup the project. + # Log some 'interesting' contextual information... + logger.info('request=%r', request) + logger.info('view=%s', view.__class__.__name__) + logger.info('view.filter_permissions=%s', view.filter_permissions) + # Get the object's content and dump it for analysis... + obj_class_name = obj.__class__.__name__ + msg = f"There is no Project at {view.filter_permissions}" + logger.error( + "%s - obj=%s vars(base_start_obj)=%s", msg, obj_class_name, vars(obj) + ) + raise PermissionDenied(msg) from exc + + if attr_value.__class__.__name__ == "ManyRelatedManager": + # Potential for many proposals... + object_proposals = [p.title for p in attr_value.all()] + else: + # Only one proposal... + object_proposals = [attr_value.title] + if not object_proposals: + raise PermissionDenied( + detail="Authority cannot be granted - the object is not a part of any Project" + ) + # Now we have the proposals the object belongs to + # has the user been associated (in IPSpyB) with any of them? + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + user=request.user, proposals=object_proposals + ): + raise PermissionDenied( + detail="Your authority to access this object has not been given" + ) + # User is a member of at least one of the object's proposals... + return True diff --git a/viewer/serializers.py b/viewer/serializers.py index 41252d89..e05b2ea7 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -1,3 +1,4 @@ +import contextlib import logging from pathlib import Path from urllib.parse import urljoin @@ -12,8 +13,9 @@ from rdkit import Chem from rdkit.Chem import Descriptors from rest_framework import serializers +from rest_framework.exceptions import PermissionDenied -from api.security import ISpyBSafeQuerySet +from api.security import ISPyBSafeQuerySet from api.utils import draw_mol, validate_tas from viewer import models from viewer.target_loader import XTALFORMS_FILE @@ -22,7 +24,91 @@ logger = logging.getLogger(__name__) -_ISPYB_SAFE_QUERY_SET = ISpyBSafeQuerySet() +_ISPYB_SAFE_QUERY_SET = ISPyBSafeQuerySet() + + +class ValidateProjectMixin: + """Mixin for serializers to check if user is allowed to create objects. + + Requires a 'filter_permissions' member in the corresponding View. + This is used to navigate to the Project object from the data map + given to the validate() method. + """ + + def validate(self, data): + # User must be logged in + user = self.context['request'].user # type: ignore [attr-defined] + if not user or not user.is_authenticated: + raise serializers.ValidationError("You must be logged in") + view = self.context['view'] # type: ignore [attr-defined] + if not hasattr(view, "filter_permissions"): + raise AttributeError( + "The view object must define a 'filter_permissions' property" + ) + + # We expect a filter_permissions string (defined in the View) like this... + # "compound__project_id" + # In this example the supplied data map is therefore expected to have a + # "compound" key (which we extract into a variable called 'base_object_key'). + # We use the 2nd half of the string (which we call 'project_path') + # to get to the Project object from 'data["compound"]'. + # + # If the filter_permissions string has no 2nd half (e.g. it's simply 'project_id') + # then the data is clearly expected to contain the Project object itself. + + base_object_key, project_path = view.filter_permissions.split('__', 1) + base_start_obj = data[base_object_key] + # Assume we're using the base object, + # but swap it out of there's a project path. + project_obj = base_start_obj + if project_path: + try: + project_obj = getattr(base_start_obj, project_path) + except AttributeError as exc: + # Something's gone wrong trying to lookup the project. + # Log some 'interesting' contextual information... + logger.info('context=%s', self.context) # type: ignore [attr-defined] + logger.info('data=%s', data) + logger.info('view=%s', view.__class__.__name__) + logger.info('view.filter_permissions=%s', view.filter_permissions) + # Get the object's content and dump it for analysis... + bso_class_name = base_start_obj.__class__.__name__ + msg = f"There is no Project at '{project_path}' ({view.filter_permissions})" + logger.error( + "%s - base_start_obj=%s vars(base_start_obj)=%s", + msg, + bso_class_name, + vars(base_start_obj), + ) + raise serializers.ValidationError(msg) from exc + assert project_obj + # Now get the proposals from the Project(s)... + if project_obj.__class__.__name__ == "ManyRelatedManager": + # Potential for many proposals... + object_proposals = [p.title for p in project_obj.all()] + else: + # Only one proposal... + object_proposals = [project_obj.title] + if not object_proposals: + raise PermissionDenied( + detail="Authority cannot be granted - the object is not a part of any Project" + ) + + # Now we have the proposals (Project titles) the object belongs to, + # has the user been associated (in IPSpyB) with any of them? + # We can always see (GET) objects that are open to the public. + restrict_public = False if self.context['request'].method == 'GET' else True # type: ignore [attr-defined] + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + user=user, + proposals=object_proposals, + restrict_public_to_membership=restrict_public, + ): + raise PermissionDenied( + detail="Your authority to access this object has not been given" + ) + + # OK if we get here... + return data class FileSerializer(serializers.ModelSerializer): @@ -37,7 +123,7 @@ class Meta: fields = '__all__' -class CompoundIdentifierSerializer(serializers.ModelSerializer): +class CompoundIdentifierSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.CompoundIdentifier fields = '__all__' @@ -48,19 +134,8 @@ class TargetSerializer(serializers.ModelSerializer): zip_archive = serializers.SerializerMethodField() metadata = serializers.SerializerMethodField() - def get_template_protein(self, obj): - exp_upload = ( - models.ExperimentUpload.objects.filter( - target=obj, - ) - .order_by('-commit_datetime') - .first() - ) - - yaml_path = exp_upload.get_upload_path() - - # last components of path, need for reconstruction later - comps = yaml_path.parts[-2:] + def get_template_protein_path(self, experiment_upload) -> Path | None: + yaml_path = experiment_upload.get_upload_path() # and the file itself yaml_path = yaml_path.joinpath(XTALFORMS_FILE) @@ -72,44 +147,56 @@ def get_template_protein(self, obj): assemblies = contents["assemblies"] except KeyError: logger.error("No 'assemblies' section in '%s'", XTALFORMS_FILE) - return '' + return None try: first = list(assemblies.values())[0] except IndexError: logger.error("No assemblies in 'assemblies' section") - return '' + return None try: reference = first["reference"] except KeyError: logger.error("No assemblies in 'assemblies' section") - return '' + return None ref_path = ( Path(settings.TARGET_LOADER_MEDIA_DIRECTORY) - .joinpath(exp_upload.task_id) - .joinpath(comps[0]) - .joinpath(comps[1]) + .joinpath(experiment_upload.target.zip_archive.name) + .joinpath(experiment_upload.upload_data_dir) .joinpath("crystallographic_files") .joinpath(reference) .joinpath(f"{reference}.pdb") ) logger.debug('ref_path: %s', ref_path) if Path(settings.MEDIA_ROOT).joinpath(ref_path).is_file(): - request = self.context.get('request', None) - if request is not None: - return request.build_absolute_uri( - Path(settings.MEDIA_URL).joinpath(ref_path) - ) - else: - return '' + return ref_path else: logger.error("Reference pdb file doesn't exist") - return '' + return None else: logger.error("'%s' missing", XTALFORMS_FILE) - return '' + return None + + def get_template_protein(self, obj): + # loop through exp uploads from latest to earliest, and try to + # find template protein + for exp_upload in models.ExperimentUpload.objects.filter( + target=obj, + ).order_by('-commit_datetime'): + path = self.get_template_protein_path(exp_upload) + if path is None: + continue + else: + request = self.context.get('request', None) + if request is not None: + return request.build_absolute_uri( + Path(settings.MEDIA_URL).joinpath(path) + ) + else: + return None + return None def get_zip_archive(self, obj): # The if-check is because the filefield in target has null=True. @@ -125,7 +212,6 @@ def get_metadata(self, obj): class Meta: model = models.Target - # TODO: it's missing protein_set. is it necessary anymore? fields = ( "id", "title", @@ -548,7 +634,7 @@ class Meta: fields = '__all__' -class ComputedSetSerializer(serializers.ModelSerializer): +class ComputedSetSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.ComputedSet fields = '__all__' @@ -657,6 +743,7 @@ class DiscoursePostWriteSerializer(serializers.Serializer): class DictToCsvSerializer(serializers.Serializer): title = serializers.CharField(max_length=200) + filename = serializers.CharField() dict = serializers.DictField() @@ -712,14 +799,14 @@ class DownloadStructuresSerializer(serializers.Serializer): # Start of Serializers for Squonk Jobs # (GET) -class JobFileTransferReadSerializer(serializers.ModelSerializer): +class JobFileTransferReadSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.JobFileTransfer fields = '__all__' # (POST, PUT, PATCH) -class JobFileTransferWriteSerializer(serializers.ModelSerializer): +class JobFileTransferWriteSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.JobFileTransfer fields = ("snapshot", "target", "squonk_project", "proteins", "compounds") @@ -767,7 +854,7 @@ class Meta: fields = ("job_status", "state_transition_time") -class TargetExperimentReadSerializer(serializers.ModelSerializer): +class TargetExperimentReadSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.ExperimentUpload fields = '__all__' @@ -821,6 +908,18 @@ class SiteObservationReadSerializer(serializers.ModelSerializer): compound_code = serializers.StringRelatedField() prefix_tooltip = serializers.StringRelatedField() + ligand_mol_file = serializers.SerializerMethodField() + + def get_ligand_mol_file(self, obj): + contents = '' + if obj.ligand_mol: + path = Path(settings.MEDIA_ROOT).joinpath(obj.ligand_mol.name) + with contextlib.suppress(TypeError, FileNotFoundError): + with open(path, "r", encoding="utf-8") as f: + contents = f.read() + + return contents + class Meta: model = models.SiteObservation fields = '__all__' @@ -844,7 +943,7 @@ class Meta: fields = '__all__' -class PoseSerializer(serializers.ModelSerializer): +class PoseSerializer(ValidateProjectMixin, serializers.ModelSerializer): site_observations = serializers.PrimaryKeyRelatedField( many=True, queryset=models.SiteObservation.objects.all(), @@ -920,7 +1019,9 @@ def validate(self, data): - the pose they're being removed from is deleted when empty """ - logger.info('+ validate: %s', data) + logger.info('+ validate data: %s', data) + + data = super().validate(data) template = ( "Site observation {} cannot be assigned to pose because " diff --git a/viewer/services.py b/viewer/services.py deleted file mode 100644 index 61fca9e4..00000000 --- a/viewer/services.py +++ /dev/null @@ -1,224 +0,0 @@ -import asyncio -import functools -import logging -import os -from concurrent import futures -from enum import Enum - -import requests -from django.conf import settings -from frag.utils.network_utils import get_driver -from pydiscourse import DiscourseClient - -from api.security import ping_configured_connector -from viewer.squonk2_agent import get_squonk2_agent - -logger = logging.getLogger(__name__) - -# Service query timeout -SERVICE_QUERY_TIMEOUT_S = 28 -# Default timeout for any request calls -# Used for keycloak atm. -REQUEST_TIMEOUT_S = 5 - -_NEO4J_LOCATION: str = settings.NEO4J_QUERY -_NEO4J_AUTH: str = settings.NEO4J_AUTH - - -# TIMEOUT is no longer used. -# A service timeout is considered a service that is degraded -class State(str, Enum): - NOT_CONFIGURED = "NOT_CONFIGURED" - DEGRADED = "DEGRADED" - OK = "OK" - TIMEOUT = "TIMEOUT" - ERROR = "ERROR" - - -class Service(str, Enum): - ISPYB = "ispyb" - DISCOURSE = "discourse" - SQUONK = "squonk" - FRAG = "fragmentation_graph" - KEYCLOAK = "keycloak" - - -# called from the outside -def get_service_state(services): - return asyncio.run(service_queries(services)) - - -async def service_queries(services): - """Chain the requested service queries""" - logger.debug("service query called") - coroutines = [] - if Service.ISPYB in services: - coroutines.append( - ispyb( - Service.ISPYB, - "Access control (ISPyB)", - ispyb_host="ISPYB_HOST", - ) - ) - - if Service.SQUONK in services: - coroutines.append( - squonk( - Service.SQUONK, - "Squonk", - squonk_pwd="SQUONK2_ORG_OWNER_PASSWORD", - ) - ) - - if Service.FRAG in services: - coroutines.append( - fragmentation_graph( - Service.FRAG, - "Fragmentation graph", - url="NEO4J_BOLT_URL", - ) - ) - - if Service.DISCOURSE in services: - coroutines.append( - discourse( - Service.DISCOURSE, - "Discourse", - key="DISCOURSE_API_KEY", - url="DISCOURSE_HOST", - user="DISCOURSE_USER", - ) - ) - - if Service.KEYCLOAK in services: - coroutines.append( - keycloak( - Service.KEYCLOAK, - "Keycloak", - url="OIDC_KEYCLOAK_REALM", - secret="OIDC_RP_CLIENT_SECRET", - ) - ) - - logger.debug("coroutines: %s", coroutines) - result = await asyncio.gather(*coroutines) - logger.debug("service query result: %s", result) - return result - - -def service_query(func): - """Decorator function for service queries functions""" - - @functools.wraps(func) - async def wrapper_service_query(*args, **kwargs): - logger.debug("+ wrapper_service_query") - logger.debug("args passed: %s", args) - logger.debug("kwargs passed: %s", kwargs) - logger.debug("function: %s", func.__name__) - - state = State.NOT_CONFIGURED - envs = [os.environ.get(k, None) for k in kwargs.values()] - # env variables come in kwargs, all must be defined - if all(envs): - state = State.DEGRADED - loop = asyncio.get_running_loop() - # memo to self: executor is absolutely crucial, otherwise - # TimeoutError is not caught - executor = futures.ThreadPoolExecutor() - try: - async with asyncio.timeout(SERVICE_QUERY_TIMEOUT_S): - future = loop.run_in_executor( - executor, functools.partial(func, *args, **kwargs) - ) - logger.debug("future: %s", future) - result = await future - logger.debug("result: %s", result) - if result: - state = State.OK - - except TimeoutError: - # Timeout is an "expected" condition for a service that's expected - # to be running but is taking too long to report its state - # and is also considered DEGRADED. - state = State.DEGRADED - except Exception as exc: - # unknown error with the query - logger.exception(exc, exc_info=True) - state = State.ERROR - - # ID and Name are the 1st and 2nd params respectively. - # Alternative solution for this would be to return just a - # state and have the service_queries() map the results to the - # correct values - if state not in [State.OK, State.NOT_CONFIGURED]: - logger.info('"%s" is %s', args[1], state.name) - return {"id": args[0], "name": args[1], "state": state} - - return wrapper_service_query - - -@service_query -def ispyb(func_id, name, ispyb_host=None) -> bool: - # Unused arguments - del func_id, name, ispyb_host - - logger.debug("+ ispyb") - return ping_configured_connector() - - -@service_query -def discourse(func_id, name, key=None, url=None, user=None) -> bool: - # Unused arguments - del func_id, name - - logger.debug("+ discourse") - # Discourse is "unconfigured" if there is no API key - if not settings.DISCOURSE_API_KEY: - return False - client = DiscourseClient( - os.environ.get(url, None), - api_username=os.environ.get(user, None), - api_key=os.environ.get(key, None), - ) - # TODO: some action on client? - return client != None - - -@service_query -def squonk(func_id, name, squonk_pwd=None) -> bool: - # Unused arguments - del func_id, name, squonk_pwd - - logger.debug("+ squonk") - return get_squonk2_agent().configured().success - - -@service_query -def fragmentation_graph(func_id, name, url=None) -> bool: - # Unused arguments - del func_id, name, url - - logger.debug("+ fragmentation_graph") - graph_driver = get_driver(url=_NEO4J_LOCATION, neo4j_auth=_NEO4J_AUTH) - with graph_driver.session() as session: - try: - _ = session.run("match (n) return count (n);") - return True - except ValueError: - # service isn't running - return False - - -@service_query -def keycloak(func_id, name, url=None, secret=None) -> bool: - # Unused arguments - del func_id, name, secret - - logger.debug("+ keycloak") - # Keycloak is "unconfigured" if there is no realm URL - keycloak_realm = os.environ.get(url, None) - if not keycloak_realm: - return False - response = requests.get(keycloak_realm, timeout=REQUEST_TIMEOUT_S) - logger.debug("keycloak response: %s", response) - return response.ok diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index aadab6c0..7d4b6e92 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -21,7 +21,7 @@ from urllib3.exceptions import InsecureRequestWarning from wrapt import synchronized -from api.security import ISpyBSafeQuerySet +from api.security import ISPyBSafeQuerySet from viewer.models import Project, Squonk2Org, Squonk2Project, Squonk2Unit, Target, User _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -133,7 +133,7 @@ def __init__(self): # Used when we are given a tas (target access string). # It allows us to check that a user is permitted to use the access ID # and relies on ISPyB credentials present in the environment. - self.__ispyb_safe_query_set: ISpyBSafeQuerySet = ISpyBSafeQuerySet() + self.__ispyb_safe_query_set: ISPyBSafeQuerySet = ISPyBSafeQuerySet() def _get_user_name(self, user_id: int) -> str: # Gets the username (if id looks sensible) @@ -723,7 +723,7 @@ def _verify_access(self, c_params: CommonParams) -> Squonk2AgentRv: target_access_string = self._get_target_access_string(access_id) assert target_access_string proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user( - user, restrict_to_membership=True + user, restrict_public_to_membership=True ) if not target_access_string in proposal_list: msg = ( diff --git a/viewer/target_loader.py b/viewer/target_loader.py index 26e3719d..0c434906 100644 --- a/viewer/target_loader.py +++ b/viewer/target_loader.py @@ -1,19 +1,17 @@ import contextlib import functools import hashlib -import itertools import logging import math import os -import string +import shutil import tarfile -import uuid from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, TypeVar +from typing import Any, Dict, Iterable, List, Optional, Tuple, TypeVar import yaml from celery import Task @@ -46,6 +44,7 @@ XtalformQuatAssembly, XtalformSite, ) +from viewer.utils import alphanumerator, clean_object_id, sanitize_directory_name logger = logging.getLogger(__name__) @@ -53,10 +52,6 @@ # assemblies and xtalforms XTALFORMS_FILE = "assemblies.yaml" -# holding horses for now -# # assigned xtalforms, not all are referenced in meta_aligner -# ASSIGNED_XTALFORMS_FILE = "assigned_xtalforms.yaml" - # target name, nothing else CONFIG_FILE = "config*.yaml" @@ -138,6 +133,7 @@ def __str__(self): @dataclass class UploadReport: task: Task | None + proposal_ref: str stack: list[UploadReportEntry] = field(default_factory=list) upload_state: UploadState = UploadState.PROCESSING failed: bool = False @@ -175,6 +171,7 @@ def _update_task(self, message: str | list) -> None: self.task.update_state( state=self.upload_state, meta={ + "proposal_ref": self.proposal_ref, "description": message, }, ) @@ -278,29 +275,6 @@ def calculate_sha256(filepath) -> str: return sha256_hash.hexdigest() -def alphanumerator(start_from: str = "") -> Generator[str, None, None]: - """Return alphabetic generator (A, B .. AA, AB...) starting from a specified point.""" - - # since product requries finite maximum return string length set - # to 10 characters. that should be enough for fragalysis (and to - # cause database issues) - generator = ( - "".join(word) - for word in itertools.chain.from_iterable( - itertools.product(string.ascii_lowercase, repeat=i) for i in range(1, 11) - ) - ) - - # Drop values until the starting point is reached - if start_from is not None and start_from != '': - start_from = start_from.lower() - generator = itertools.dropwhile(lambda x: x != start_from, generator) # type: ignore[assignment] - # and drop one more, then it starts from after the start from as it should - _ = next(generator) - - return generator - - def strip_version(s: str, separator: str = "/") -> Tuple[str, int]: # format something like XX01ZVNS2B-x0673/B/501/1 # remove tailing '1' @@ -362,13 +336,15 @@ def wrapper_create_objects( ) obj.save() new = True + except MultipleObjectsReturned: + msg = "{}.get_or_create in {} returned multiple objects for {}".format( + instance_data.model_class._meta.object_name, # pylint: disable=protected-access + instance_data.key, + instance_data.fields, + ) + self.report.log(logging.ERROR, msg) + failed = failed + 1 - # obj, new = instance_data.model_class.filter_manager.by_target( - # self.target - # ).get_or_create( - # **instance_data.fields, - # defaults=instance_data.defaults, - # ) else: # no unique field requirements, just create new object obj = instance_data.model_class( @@ -381,15 +357,6 @@ def wrapper_create_objects( instance_data.model_class._meta.object_name, # pylint: disable=protected-access obj, ) - - except MultipleObjectsReturned: - msg = "{}.get_or_create in {} returned multiple objects for {}".format( - instance_data.model_class._meta.object_name, # pylint: disable=protected-access - instance_data.key, - instance_data.fields, - ) - self.report.log(logging.ERROR, msg) - failed = failed + 1 except IntegrityError: msg = "{} object {} failed to save".format( instance_data.model_class._meta.object_name, # pylint: disable=protected-access @@ -440,15 +407,20 @@ def wrapper_create_objects( # index data here probs result[instance_data.versioned_key] = m - msg = "{} {} objects processed, {} created, {} fetched from database".format( - created + existing + failed, - next( # pylint: disable=protected-access - iter(result.values()) - ).instance._meta.model._meta.object_name, # pylint: disable=protected-access - created, - existing, - ) # pylint: disable=protected-access - self.report.log(logging.INFO, msg) + if result: + msg = "{} {} objects processed, {} created, {} fetched from database".format( + created + existing + failed, + next( # pylint: disable=protected-access + iter(result.values()) + ).instance._meta.model._meta.object_name, # pylint: disable=protected-access + created, + existing, + ) # pylint: disable=protected-access + self.report.log(logging.INFO, msg) + else: + # cannot continue when one object type is missing, abort + msg = f"No objects returned by {func.__name__}" + self.report.log(logging.ERROR, msg) # refresh all objects to make sure they're up to date. # this is specifically because of the superseded flag above - @@ -489,7 +461,7 @@ def __init__( self.previous_version_dirs = None self.user_id = user_id - self.report = UploadReport(task=task) + self.report = UploadReport(task=task, proposal_ref=self.proposal_ref) self.raw_data.mkdir() @@ -501,29 +473,23 @@ def __init__( ) # work out where the data finally lands - # path = Path(settings.MEDIA_ROOT).joinpath(TARGET_LOADER_DATA) path = Path(TARGET_LOADER_MEDIA_DIRECTORY) - # give each upload a unique directory. since I already have - # task_id, why not reuse it + # give each upload a unique directory + # update: resolving issue 1311 introduced a bug, where + # subsequent uploads overwrote file paths and files appeared + # to be missing. changing the directory structure so this + # wouldn't be an issue, the new structure is + # target_loader_data/target_title/upload_(n)/... if task: - path = path.joinpath(str(task.request.id)) self.experiment_upload.task_id = task.request.id - else: - # unless of course I don't have task.. - # TODO: i suspect this will never be used. - path_uuid = uuid.uuid4().hex - path = path.joinpath(path_uuid) - self.experiment_upload.task_id = path_uuid # figure out absolute and relative paths to final # location. relative path is added to db field, this will be # used in url requests to retrieve the file. absolute path is # for moving the file to the final location - self._final_path = path.joinpath(self.bundle_name) - self._abs_final_path = ( - Path(settings.MEDIA_ROOT).joinpath(path).joinpath(self.bundle_name) - ) + self._final_path = path + self._abs_final_path = Path(settings.MEDIA_ROOT).joinpath(path) # but don't create now, this comes later # to be used in logging messages, if no task, means invoked @@ -872,6 +838,7 @@ def process_experiment( "cif_info": str(self._get_final_path(cif_info)), "map_info": map_info_paths, "prefix_tooltip": prefix_tooltip, + "code_prefix": code_prefix, # this doesn't seem to be present # pdb_sha256: } @@ -1109,6 +1076,7 @@ def process_canon_site( Incoming data format: : + centroid_res: conformer_site_ids: global_reference_dtag: reference_conformer_site_id: @@ -1132,6 +1100,11 @@ def process_canon_site( ) residues = extract(key="residues", return_type=list) + centroid_res = extract(key="centroid_res") + conf_sites_ids = extract(key="conformer_site_ids", return_type=list) + ref_conf_site_id = extract(key="reference_conformer_site_id") + + centroid_res = f"{centroid_res}_v{version}" fields = { "name": canon_site_id, @@ -1140,11 +1113,9 @@ def process_canon_site( defaults = { "residues": residues, + "centroid_res": centroid_res, } - conf_sites_ids = extract(key="conformer_site_ids", return_type=list) - ref_conf_site_id = extract(key="reference_conformer_site_id") - index_data = { "ref_conf_site": ref_conf_site_id, "conformer_site_ids": conf_sites_ids, @@ -1329,7 +1300,6 @@ def process_site_observation( # wrong data item return None - idx, _ = strip_version(v_idx, separator="+") extract = functools.partial( self._extract, data=data, @@ -1340,7 +1310,10 @@ def process_site_observation( experiment = experiments[experiment_id].instance - longcode = f"{experiment.code}_{chain}_{str(ligand)}_{str(idx)}" + longcode = ( + # f"{experiment.code}_{chain}_{str(ligand)}_{str(version)}_{str(v_idx)}" + f"{experiment.code}_{chain}_{str(ligand)}_v{str(version)}" + ) key = f"{experiment.code}/{chain}/{str(ligand)}" v_key = f"{experiment.code}/{chain}/{str(ligand)}/{version}" @@ -1383,7 +1356,6 @@ def process_site_observation( apo_desolv_file, apo_file, artefacts_file, - ligand_mol_file, sigmaa_file, diff_file, event_file, @@ -1401,7 +1373,6 @@ def process_site_observation( ), recommended=( "artefacts", - "ligand_mol", "sigmaa_map", # NB! keys in meta_aligner not yet updated "diff_map", # NB! keys in meta_aligner not yet updated "event_map", @@ -1412,18 +1383,6 @@ def process_site_observation( validate_files=validate_files, ) - logger.debug('looking for ligand_mol: %s', ligand_mol_file) - - mol_data = None - if ligand_mol_file: - with contextlib.suppress(TypeError, FileNotFoundError): - with open( - self.raw_data.joinpath(ligand_mol_file), - "r", - encoding="utf-8", - ) as f: - mol_data = f.read() - fields = { # Code for this protein (e.g. Mpro_Nterm-x0029_A_501_0) "longcode": longcode, @@ -1450,7 +1409,6 @@ def process_site_observation( "ligand_mol": str(self._get_final_path(ligand_mol)), "ligand_smiles": str(self._get_final_path(ligand_smiles)), "pdb_header_file": "currently missing", - "ligand_mol_file": mol_data, } return ProcessedObject( @@ -1501,7 +1459,7 @@ def process_bundle(self): xtalforms_yaml = self._load_yaml(Path(upload_dir).joinpath(XTALFORMS_FILE)) # this is the last file to load. if any of the files missing, don't continue - if not meta or not config or not xtalforms_yaml: + if not any([meta, config, xtalforms_yaml]): msg = "Missing files in uploaded data, aborting" raise FileNotFoundError(msg) @@ -1524,6 +1482,21 @@ def process_bundle(self): display_name=self.target_name, ) + if target_created: + # mypy thinks target and target_name are None + target_dir = sanitize_directory_name(self.target_name, self.abs_final_path) # type: ignore [arg-type] + self.target.zip_archive = target_dir # type: ignore [attr-defined] + self.target.save() # type: ignore [attr-defined] + else: + # NB! using existing field zip_archive to point to the + # location of the archives, not the archives + # themselves. The field was unused, and because of the + # versioned uploads, there's no single archive anymore + target_dir = str(self.target.zip_archive) # type: ignore [attr-defined] + + self._final_path = self._final_path.joinpath(target_dir) + self._abs_final_path = self._abs_final_path.joinpath(target_dir) + # TODO: original target loader's function get_create_projects # seems to handle more cases. adopt or copy visit = self.proposal_ref.split()[0] @@ -1660,7 +1633,7 @@ def process_bundle(self): ) canon_site_objects = self.process_canon_site(yaml_data=canon_sites) - self._enumerate_objects(canon_site_objects, "canon_site_num") + # NB! missing fk's: # - ref_conf_site # - quat_assembly @@ -1684,27 +1657,8 @@ def process_bundle(self): canon_sites=canon_sites_by_conf_sites, xtalforms=xtalform_objects, ) - # enumerate xtalform_sites. a bit trickier than others because - # requires alphabetic enumeration - last_xtsite = ( - XtalformSite.objects.filter( - pk__in=[ - k.instance.pk - for k in xtalform_sites_objects.values() # pylint: disable=no-member - ] - ) - .order_by("-xtalform_site_num")[0] - .xtalform_site_num - ) - - xtnum = alphanumerator(start_from=last_xtsite) - for val in xtalform_sites_objects.values(): # pylint: disable=no-member - if not val.instance.xtalform_site_num: - val.instance.xtalform_site_num = next(xtnum) - val.instance.save() # now can update CanonSite with ref_conf_site - # also, fill the canon_site_num field # TODO: ref_conf_site is with version, object's key isn't for val in canon_site_objects.values(): # pylint: disable=no-member val.instance.ref_conf_site = canon_site_conf_objects[ @@ -1790,7 +1744,8 @@ def process_bundle(self): ] # iter_pos = next(suffix) # code = f"{code_prefix}{so.experiment.code.split('-')[1]}{iter_pos}" - code = f"{code_prefix}{so.experiment.code.split('-')[1]}{next(suffix)}" + # code = f"{code_prefix}{so.experiment.code.split('-')[1]}{next(suffix)}" + code = f"{code_prefix}{so.experiment.code.split('-x')[1]}{next(suffix)}" # test uniqueness for target # TODO: this should ideally be solved by db engine, before @@ -1835,35 +1790,98 @@ def process_bundle(self): logger.debug("data read and processed, adding tags") # tag site observations - for val in canon_site_objects.values(): # pylint: disable=no-member + cat_canon = TagCategory.objects.get(category="CanonSites") + # sort canon sites by number of observations + # fmt: off + canon_sort_qs = CanonSite.objects.filter( + pk__in=[k.instance.pk for k in canon_site_objects.values() ], # pylint: disable=no-member + ).annotate( + # obvs=Count("canonsiteconf_set__siteobservation_set", default=0), + obvs=Count("canonsiteconf__siteobservation", default=0), + ).order_by("-obvs", "name") + # ordering by name is not strictly necessary, but + # makes the sorting consistent + + # fmt: on + + logger.debug('canon_site_order') + for site in canon_sort_qs: + logger.debug('%s: %s', site.name, site.obvs) + + _canon_site_objects = {} + for site in canon_sort_qs: + key = f"{site.name}+{site.version}" + _canon_site_objects[key] = canon_site_objects[ + key + ] # pylint: disable=no-member + + self._enumerate_objects(_canon_site_objects, "canon_site_num") + for val in _canon_site_objects.values(): # pylint: disable=no-member prefix = val.instance.canon_site_num - tag = ''.join(val.instance.name.split('+')[1:-1]) + # tag = canon_name_tag_map.get(val.versioned_key, "UNDEFINED") so_list = SiteObservation.objects.filter( canon_site_conf__canon_site=val.instance ) - self._tag_observations(tag, prefix, "CanonSites", so_list) + tag = val.versioned_key + try: + short_tag = val.versioned_key.split('-')[1][1:] + main_obvs = val.instance.ref_conf_site.ref_site_observation + code_prefix = experiment_objects[main_obvs.experiment.code].index_data[ + "code_prefix" + ] + short_tag = f"{code_prefix}{short_tag}" + except IndexError: + short_tag = tag + + self._tag_observations( + tag, + prefix, + category=cat_canon, + site_observations=so_list, + short_tag=short_tag, + ) logger.debug("canon_site objects tagged") numerators = {} - for val in canon_site_conf_objects.values(): # pylint: disable=no-member + cat_conf = TagCategory.objects.get(category="ConformerSites") + for val in canon_site_conf_objects.values(): # pylint: + # disable=no-member problem introduced with the sorting of + # canon sites (issue 1498). objects somehow go out of sync + val.instance.refresh_from_db() if val.instance.canon_site.canon_site_num not in numerators.keys(): numerators[val.instance.canon_site.canon_site_num] = alphanumerator() prefix = ( f"{val.instance.canon_site.canon_site_num}" + f"{next(numerators[val.instance.canon_site.canon_site_num])}" ) - tag = val.instance.name.split('+')[0] so_list = [ - site_observation_objects[k].instance - for k in val.index_data["members"] - # site_observations_versioned[k] - # for k in val.index_data["members"] + site_observation_objects[k].instance for k in val.index_data["members"] ] - self._tag_observations(tag, prefix, "ConformerSites", so_list) + # tag = val.instance.name.split('+')[0] + tag = val.instance.name + try: + short_tag = val.instance.name.split('-')[1][1:] + main_obvs = val.instance.ref_site_observation + code_prefix = experiment_objects[main_obvs.experiment.code].index_data[ + "code_prefix" + ] + short_tag = f"{code_prefix}{short_tag}" + except IndexError: + short_tag = tag + + self._tag_observations( + tag, + prefix, + category=cat_conf, + site_observations=so_list, + hidden=True, + short_tag=short_tag, + ) logger.debug("conf_site objects tagged") + cat_quat = TagCategory.objects.get(category="Quatassemblies") for val in quat_assembly_objects.values(): # pylint: disable=no-member prefix = f"A{val.instance.assembly_num}" tag = val.instance.name @@ -1872,30 +1890,107 @@ def process_bundle(self): quat_assembly=val.instance ).values("xtalform") ) - self._tag_observations(tag, prefix, "Quatassemblies", so_list) + self._tag_observations( + tag, prefix, category=cat_quat, site_observations=so_list + ) logger.debug("quat_assembly objects tagged") + cat_xtal = TagCategory.objects.get(category="Crystalforms") for val in xtalform_objects.values(): # pylint: disable=no-member prefix = f"F{val.instance.xtalform_num}" - tag = val.instance.name so_list = SiteObservation.objects.filter( xtalform_site__xtalform=val.instance ) - self._tag_observations(tag, prefix, "Crystalforms", so_list) + tag = val.instance.name + + self._tag_observations( + tag, prefix, category=cat_xtal, site_observations=so_list + ) logger.debug("xtalform objects tagged") - for val in xtalform_sites_objects.values(): # pylint: disable=no-member + # enumerate xtalform_sites. a bit trickier than others because + # requires alphabetic enumeration starting from the letter of + # the chain and following from there + + # sort the dictionary + # fmt: off + xtls_sort_qs = XtalformSite.objects.filter( + pk__in=[k.instance.pk for k in xtalform_sites_objects.values() ], # pylint: disable=no-member + ).annotate( + obvs=Count("canon_site__canonsiteconf__siteobservation", default=0), + ).order_by("-obvs", "xtalform_site_id") + # ordering by xtalform_site_id is not strictly necessary, but + # makes the sorting consistent + + # fmt: on + + _xtalform_sites_objects = {} + for xtl in xtls_sort_qs: + key = f"{xtl.xtalform_site_id}/{xtl.version}" + _xtalform_sites_objects[key] = xtalform_sites_objects[ + key + ] # pylint: disable=no-member + + if self.version_number == 1: + # first upload, use the chain letter + xtnum = alphanumerator( + start_from=xtls_sort_qs[0].lig_chain.lower(), drop_first=False + ) + else: + # subsequent upload, just use the latest letter as starting point + # fmt: off + last_xtsite = XtalformSite.objects.filter( + pk__in=[ + k.instance.pk + for k in _xtalform_sites_objects.values() # pylint: disable=no-member + ] + ).order_by( + "-xtalform_site_num" + )[0].xtalform_site_num + # fmt: on + xtnum = alphanumerator(start_from=last_xtsite) + + # this should be rare, as Frank said, all crystal-related + # issues should be resolved by the time of the first + # upload. In fact, I'll mark this momentous occasion here: + logger.warning("New XtalformSite objects added in subsequent uploads") + + for val in _xtalform_sites_objects.values(): # pylint: disable=no-member + if not val.instance.xtalform_site_num: + val.instance.xtalform_site_num = next(xtnum) + val.instance.save() + + cat_xtalsite = TagCategory.objects.get(category="CrystalformSites") + for val in _xtalform_sites_objects.values(): # pylint: disable=no-member prefix = ( f"F{val.instance.xtalform.xtalform_num}" + f"{val.instance.xtalform_site_num}" ) - tag = f"{val.instance.xtalform.name} - {val.instance.xtalform_site_id}" so_list = [ site_observation_objects[k].instance for k in val.index_data["residues"] ] - self._tag_observations(tag, prefix, "CrystalformSites", so_list) + tag = val.versioned_key + try: + # remove protein name and 'x' + short_tag = val.instance.xtalform_site_id.split('-')[1][1:] + main_obvs = val.instance.canon_site.ref_conf_site.ref_site_observation + code_prefix = experiment_objects[main_obvs.experiment.code].index_data[ + "code_prefix" + ] + short_tag = f"{code_prefix}{short_tag}" + except IndexError: + short_tag = tag + + self._tag_observations( + tag, + prefix, + category=cat_xtalsite, + site_observations=so_list, + hidden=True, + short_tag=short_tag, + ) logger.debug("xtalform_sites objects tagged") @@ -1906,7 +2001,7 @@ def process_bundle(self): self._tag_observations( "New", "", - "Other", + TagCategory.objects.get(category="Other"), [ k.instance for k in site_observation_objects.values() # pylint: disable=no-member @@ -1914,8 +2009,8 @@ def process_bundle(self): ], ) - def _load_yaml(self, yaml_file: Path) -> dict | None: - contents = None + def _load_yaml(self, yaml_file: Path) -> dict: + contents = {} try: with open(yaml_file, "r", encoding="utf-8") as file: contents = yaml.safe_load(file) @@ -1965,7 +2060,9 @@ def _extract( def _generate_poses(self): values = ["canon_site_conf__canon_site", "cmpd"] # fmt: off - pose_groups = SiteObservation.objects.exclude( + pose_groups = SiteObservation.filter_manager.by_target( + self.target, + ).exclude( canon_site_conf__canon_site__isnull=True, ).exclude( cmpd__isnull=True, @@ -1998,25 +2095,38 @@ def _generate_poses(self): pose.save() except MultipleObjectsReturned: # must be a follow-up upload. create new pose, but - # only add observatons that are not yet assigned + # only add observatons that are not yet assigned (if + # these exist) pose_items = pose_items.filter(pose__isnull=True) - sample = pose_items.first() - pose = Pose( - canon_site=sample.canon_site_conf.canon_site, - compound=sample.cmpd, - main_site_observation=sample, - display_name=sample.code, - ) - pose.save() + if pose_items.exists(): + sample = pose_items.first() + pose = Pose( + canon_site=sample.canon_site_conf.canon_site, + compound=sample.cmpd, + main_site_observation=sample, + display_name=sample.code, + ) + pose.save() + else: + # I don't know if this can happen but this (due to + # other bugs) is what allowed me to find this + # error. Make a note in the logs. + logger.warning("No observations left to assign to pose") # finally add observations to the (new or existing) pose for obvs in pose_items: obvs.pose = pose obvs.save() - self._tag_observations(pose.display_name, "P", "Pose", pose_items) - - def _tag_observations(self, tag, prefix, category, so_list): + def _tag_observations( + self, + tag: str, + prefix: str, + category: TagCategory, + site_observations: list, + hidden: bool = False, + short_tag: str | None = None, + ) -> None: try: # memo to self: description is set to tag, but there's # no fk to tag, instead, tag has a fk to @@ -2043,6 +2153,13 @@ def _tag_observations(self, tag, prefix, category, so_list): so_group.save() name = f"{prefix} - {tag}" if prefix else tag + tag = tag if short_tag is None else short_tag + short_name = name if short_tag is None else f"{prefix} - {short_tag}" + + tag = clean_object_id(tag) + name = clean_object_id(name) + short_name = clean_object_id(short_name) + try: so_tag = SiteObservationTag.objects.get( upload_name=name, target=self.target @@ -2052,18 +2169,21 @@ def _tag_observations(self, tag, prefix, category, so_list): # changing anything. so_tag.mol_group = so_group except SiteObservationTag.DoesNotExist: - so_tag = SiteObservationTag() - so_tag.tag = tag - so_tag.tag_prefix = prefix - so_tag.upload_name = name - so_tag.category = TagCategory.objects.get(category=category) - so_tag.target = self.target - so_tag.mol_group = so_group + so_tag = SiteObservationTag( + tag=tag, + tag_prefix=prefix, + upload_name=name, + category=category, + target=self.target, + mol_group=so_group, + hidden=hidden, + short_tag=short_name, + ) so_tag.save() - so_group.site_observation.add(*so_list) - so_tag.site_observations.add(*so_list) + so_group.site_observation.add(*site_observations) + so_tag.site_observations.add(*site_observations) def _is_already_uploaded(self, target_created, project_created): if target_created or project_created: @@ -2154,8 +2274,16 @@ def load_target( def _move_and_save_target_experiment(target_loader): # Move the uploaded file to its final location - target_loader.abs_final_path.mkdir(parents=True) - target_loader.raw_data.rename(target_loader.abs_final_path) + try: + target_loader.abs_final_path.mkdir(parents=True) + except FileExistsError: + # subsequent upload, directory already exists + pass + + shutil.move( + str(target_loader.raw_data.joinpath(target_loader.version_dir)), + str(target_loader.abs_final_path), + ) Path(target_loader.bundle_path).rename( target_loader.abs_final_path.parent.joinpath(target_loader.data_bundle) ) diff --git a/viewer/tasks.py b/viewer/tasks.py index cd360587..2274fdd4 100644 --- a/viewer/tasks.py +++ b/viewer/tasks.py @@ -86,6 +86,7 @@ def process_compound_set(validate_output): logger.warning('process_compound_set() EXIT params=%s (not validated)', params) return process_stage, validate_dict, validated + computed_set_name = params.get('update', None) submitter_name, submitter_method, blank_version = blank_mol_vals(params['sdf']) zfile, zfile_hashvals = PdbOps().run(params) @@ -100,6 +101,7 @@ def process_compound_set(validate_output): version=blank_version, zfile=zfile, zfile_hashvals=zfile_hashvals, + computed_set_name=computed_set_name, ) compound_set = save_mols.task() @@ -186,6 +188,7 @@ def validate_compound_set(task_params): 'sdf': sdf_file, 'target': target, 'pdb_zip': zfile, + 'update': update, } # Protect ourselves from an empty, blank or missing SD file. diff --git a/viewer/templates/viewer/react_temp.html b/viewer/templates/viewer/react_temp.html index 64bfe533..139de1b7 100644 --- a/viewer/templates/viewer/react_temp.html +++ b/viewer/templates/viewer/react_temp.html @@ -15,6 +15,9 @@ {% if user.is_authenticated %} var DJANGO_CONTEXT ={ + {% if target_warning_message %} + target_warning_message: '{{ target_warning_message }}', + {% endif %} legacy_url: '{{ legacy_url }}', username: '{{ user.username }}', email: '{{ user.email|default:"noemail" }}', @@ -28,6 +31,9 @@ } {% else %} var DJANGO_CONTEXT = { + {% if target_warning_message %} + target_warning_message: '{{ target_warning_message }}', + {% endif %} legacy_url: '{{ legacy_url }}', username: 'NOT_LOGGED_IN', email: "noemail", diff --git a/viewer/urls.py b/viewer/urls.py index f78a32d5..fe7e909f 100644 --- a/viewer/urls.py +++ b/viewer/urls.py @@ -6,7 +6,7 @@ urlpatterns = [ re_path(r"^react/*", views.react, name="react"), - path("upload_cset/", views.UploadCSet.as_view(), name="upload_cset"), + path("upload_cset/", views.UploadComputedSetView.as_view(), name="upload_cset"), path( "validate_task//", views.ValidateTaskView.as_view(), @@ -24,12 +24,14 @@ ), path("img_from_smiles/", views.img_from_smiles, name="img_from_smiles"), path("highlight_mol_diff/", views.highlight_mol_diff, name="highlight_mol_diff"), - path("sim_search/", views.similarity_search, name="sim_search"), path("open_targets/", views.get_open_targets, name="get_open_targets"), - path("compound_set//", views.cset_download, name="compound_set"), - path("protein_set//", views.pset_download, name="protein_set"), - path("upload_designs/", views.DSetUploadView.as_view(), name="upload_designs"), + path("compound_set//", views.computed_set_download, name="compound_set"), + path("upload_designs/", views.DesignSetUploadView.as_view(), name="upload_designs"), path("job_access/", views.JobAccessView.as_view(), name="job_access"), - path("task_status//", views.TaskStatus.as_view(), name="task_status"), - path("service_state/", views.ServiceState.as_view(), name="service_state"), + path( + "task_status//", + views.TaskStatusView.as_view(), + name="task_status", + ), + path("service_state/", views.ServiceStateView.as_view(), name="service_state"), ] diff --git a/viewer/utils.py b/viewer/utils.py index a2df571b..748f5530 100644 --- a/viewer/utils.py +++ b/viewer/utils.py @@ -1,25 +1,51 @@ -""" -utils.py - -Collection of technical methods tidied up in one location. -""" import fnmatch +import itertools +import json +import logging import os +import re import shutil +import string import tempfile +import uuid from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Generator, Optional from urllib.parse import urlparse +import pandas as pd from django.conf import settings +from django.contrib.auth.models import User +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from django.core.mail import send_mail +from django.db import IntegrityError, transaction +from django.db.models import F +from django.http import JsonResponse from rdkit import Chem +from scoring.models import SiteObservationGroup, SiteObvsSiteObservationGroup + +from .models import ( + SiteObservation, + SiteObservationTag, + SiteObvsSiteObservationTag, + Target, +) + +logger = logging.getLogger(__name__) + # Set .sdf file format version # Used at the start of every SDF file. SDF_VERSION = 'ver_1.2' SDF_RECORD_SEPARATOR = '$$$$\n' +# The root of all files constructed by 'dicttocsv'. +# The directory must not contain anything but dicttocsv-generated files. +# It certainly must not be the root of the media directory or any other directory in it. +# Introduced during 1247 security review. +CSV_TO_DICT_DOWNLOAD_ROOT = os.path.join(settings.MEDIA_ROOT, 'downloads', 'dicttocsv') + def is_url(url: Optional[str]) -> bool: try: @@ -177,3 +203,375 @@ def handle_uploaded_file(path: Path, f): with open(path, "wb+") as destination: for chunk in f.chunks(4096): destination.write(chunk) + + +def dump_curated_tags(filename: str) -> None: + # fmt: off + curated_tags = SiteObservationTag.objects.filter( + user__isnull=False, + ).annotate( + ann_target_name=F('target__title'), + ) + users = User.objects.filter( + pk__in=curated_tags.values('user'), + ) + siteobs_tag_group = SiteObvsSiteObservationTag.objects.filter( + site_obvs_tag__in=curated_tags.values('pk'), + ).annotate( + ann_site_obvs_longcode=F('site_observation__longcode') + ) + + site_obvs_group = SiteObservationGroup.objects.filter( + pk__in=curated_tags.values('mol_group'), + ).annotate( + ann_target_name=F('target__title'), + ) + + site_obvs_obvs_group = SiteObvsSiteObservationGroup.objects.filter( + site_obvs_group__in=site_obvs_group.values('pk'), + ).annotate( + ann_site_obvs_longcode=F('site_observation__longcode') + ) + # fmt: on + + result = {} + for qs in ( + users, + curated_tags, + siteobs_tag_group, + site_obvs_group, + site_obvs_obvs_group, + ): + if qs.exists(): + jq = JsonResponse(list(qs.values()), safe=False) + # have to pass through JsonResponse because that knows how + # to parse django db field types + data = json.loads(jq.content) + name = qs[0]._meta.label # pylint: disable=protected-access + result[name] = data + + with open(filename, 'w', encoding='utf-8') as writer: + writer.write(json.dumps(result, indent=4)) + + +def restore_curated_tags(filename: str) -> None: + with open(filename, 'r', encoding='utf-8') as reader: + content = json.loads(reader.read()) + + # models have to be saved in this order: + # 1) User + # 1) SiteObservationGroup <- target + # 2) SiteObservationTag <- target, user + # 3) SiteObvsSiteObservationGroup <- siteobvs + # 3) SiteObvsSiteObservationTag <- siteobvs + + # takes a bit different approach with target and user - if user is + # missing, restores the user and continues with tags, if target is + # missing, skips the tag. This seems logical (at least at the time + # writing this): if target hasn't been added obviously user + # doesn't care about restoring the tags, but user might be + # legitimately missing (hasn't logged in yet, and somebody else is + # uploading the data) + + targets = Target.objects.all() + site_observations = SiteObservation.objects.all() + + try: + with transaction.atomic(): + new_mol_groups_by_old_pk = {} + new_tags_by_old_pk = {} + new_users_by_old_pk = {} + + user_data = content.get( + User._meta.label, # pylint: disable=protected-access + [], + ) + for data in user_data: + pk = data.pop('id') + try: + user = User.objects.get(username=data['username']) + except User.DoesNotExist: + user = User(**data) + user.save() + + new_users_by_old_pk[pk] = user + + so_group_data = content.get( + SiteObservationGroup._meta.label, # pylint: disable=protected-access + [], + ) + for data in so_group_data: + try: + target = targets.get(title=data['ann_target_name']) + except Target.DoesNotExist: + logger.warning( + 'Tried to restore SiteObservationGroup for target that does not exist: %s', + data['ann_target_name'], + ) + continue + + data['target'] = target + pk = data.pop('id') + del data['ann_target_name'] + del data['target_id'] + sog = SiteObservationGroup(**data) + sog.save() + new_mol_groups_by_old_pk[pk] = sog + + so_tag_data = content.get( + SiteObservationTag._meta.label, # pylint: disable=protected-access + [], + ) + for data in so_tag_data: + try: + target = targets.get(title=data['ann_target_name']) + except Target.DoesNotExist: + logger.warning( + 'Tried to restore SiteObservationTag for target that does not exist: %s', + data['ann_target_name'], + ) + continue + data['target'] = target + pk = data.pop('id') + del data['ann_target_name'] + del data['target_id'] + if data['mol_group_id']: + data['mol_group_id'] = new_mol_groups_by_old_pk[ + data['mol_group_id'] + ].pk + data['user'] = new_users_by_old_pk[data['user_id']] + del data['user_id'] + tag = SiteObservationTag(**data) + try: + with transaction.atomic(): + tag.save() + except IntegrityError: + # this is an incredibly unlikely scenario where + # tag already exists - user must have, before + # restoring the tags, slightly edited an + # auto-generated tag. I can update the curated + # fields, but given they're both curated at this + # point, I choose to do nothing, skip the tag + logger.error( + 'Curated tag %s already exists, skipping restore', data['tag'] + ) + continue + + new_tags_by_old_pk[pk] = tag + + so_so_group_data = content.get( + SiteObvsSiteObservationGroup._meta.label, # pylint: disable=protected-access + [], + ) + for data in so_so_group_data: + try: + site_obvs = site_observations.get( + longcode=data['ann_site_obvs_longcode'] + ) + except SiteObservation.DoesNotExist: + logger.warning( + 'Tried to restore SiteObvsSiteObservationGroup for site_observation that does not exist: %s', + data['ann_site_obvs_longcode'], + ) + continue + site_obvs = site_observations.get( + longcode=data['ann_site_obvs_longcode'] + ) + data['site_observation'] = site_obvs + del data['id'] + del data['ann_site_obvs_longcode'] + del data['site_observation_id'] + data['site_obvs_group'] = new_mol_groups_by_old_pk[ + data['site_obvs_group_id'] + ] + del data['site_obvs_group_id'] + SiteObvsSiteObservationGroup(**data).save() + + so_so_tag_data = content.get( + SiteObvsSiteObservationTag._meta.label, # pylint: disable=protected-access + [], + ) + for data in so_so_tag_data: + try: + site_obvs = site_observations.get( + longcode=data['ann_site_obvs_longcode'] + ) + except SiteObservation.DoesNotExist: + logger.warning( + 'Tried to restore SiteObvsSiteObservationTag for site_observation that does not exist: %s', + data['ann_site_obvs_longcode'], + ) + continue + data['site_observation'] = site_obvs + del data['id'] + del data['ann_site_obvs_longcode'] + del data['site_observation_id'] + data['site_obvs_tag'] = new_tags_by_old_pk.get( + data['site_obvs_tag_id'], None + ) + del data['site_obvs_tag_id'] + if data['site_obvs_tag']: + # tag may be missing if not restored + SiteObvsSiteObservationTag(**data).save() + + except IntegrityError as exc: + logger.error(exc) + + +def alphanumerator( + start_from: str = "", drop_first: bool = True +) -> Generator[str, None, None]: + """Return alphabetic generator (A, B .. AA, AB...) starting from a specified point. + + drop_first - as per workflow, usually it's given the last letter + of previous sequence so the the next in the pipeline should be + start_from + 1. drop_first = False indicates this is not necessary + and start_from will be the first the iterator produces + + """ + + # since product requries finite maximum return string length set + # to 10 characters. that should be enough for fragalysis (and to + # cause database issues) + generator = ( + "".join(word) + for word in itertools.chain.from_iterable( + itertools.product(string.ascii_lowercase, repeat=i) for i in range(1, 11) + ) + ) + + # Drop values until the starting point is reached + if start_from is not None and start_from != '': + start_from = start_from.lower() + generator = itertools.dropwhile(lambda x: x != start_from, generator) # type: ignore[assignment] + if drop_first: + # drop one more, then it starts from after the start from as it should + _ = next(generator) + + return generator + + +def save_tmp_file(myfile): + """Save file in temporary location for validation/upload processing""" + + name = myfile.name + path = default_storage.save('tmp/' + name, ContentFile(myfile.read())) + return str(os.path.join(settings.MEDIA_ROOT, path)) + + +def create_csv_from_dict(input_dict, title=None, filename=None): + """Write a CSV file containing data from an input dictionary and return a full + to the file (in the media directory). + """ + if not filename: + filename = 'download' + + unique_dir = str(uuid.uuid4()) + download_path = os.path.join(CSV_TO_DICT_DOWNLOAD_ROOT, unique_dir) + os.makedirs(download_path, exist_ok=True) + + download_file = os.path.join(download_path, filename) + + # Remove file if it already exists + if os.path.isfile(download_file): + os.remove(download_file) + + with open(download_file, "w", newline='', encoding='utf-8') as csvfile: + if title: + csvfile.write(title) + csvfile.write("\n") + + df = pd.DataFrame.from_dict(input_dict) + df.to_csv(download_file, mode='a', header=True, index=False) + + return download_file + + +def email_task_completion( + contact_email, message_type, target_name, target_path=None, task_id=None +): + """Notify user of upload completion""" + + logger.info('+ email_notify_task_completion: ' + message_type + ' ' + target_name) + email_from = settings.EMAIL_HOST_USER + + if contact_email == '' or not email_from: + # Only send email if configured. + return + + if message_type == 'upload-success': + subject = 'Fragalysis: Target: ' + target_name + ' Uploaded' + message = ( + 'The upload of your target data is complete. Your target is available at: ' + + str(target_path) + ) + elif message_type == 'validate-success': + subject = 'Fragalysis: Target: ' + target_name + ' Validation' + message = ( + 'Your data was validated. It can now be uploaded using the upload option.' + ) + else: + # Validation failure + subject = 'Fragalysis: Target: ' + target_name + ' Validation/Upload Failed' + message = ( + 'The validation/upload of your target data did not complete successfully. ' + 'Please navigate the following link to check the errors: validate_task/' + + str(task_id) + ) + + recipient_list = [ + contact_email, + ] + logger.info('+ email_notify_task_completion email_from: %s', email_from) + logger.info('+ email_notify_task_completion subject: %s', subject) + logger.info('+ email_notify_task_completion message: %s', message) + logger.info('+ email_notify_task_completion contact_email: %s', contact_email) + + # Send email - this should not prevent returning to the screen in the case of error. + send_mail(subject, message, email_from, recipient_list, fail_silently=True) + logger.info('- email_notify_task_completion') + return + + +def sanitize_directory_name(name: str, path: Path | None = None) -> str: + """ + Sanitize a string to ensure it only contains characters allowed in UNIX directory names. + + Parameters: + name: The input string to sanitize. + path (optional): the parent directory where the directory would reside, to check if unique + + Returns: + str: A sanitized string with only allowed characters. + """ + # Define allowed characters regex + allowed_chars = re.compile(r'[^a-zA-Z0-9._-]') + + # Replace disallowed characters with an underscore + sanitized_name = allowed_chars.sub('_', name.strip()) + + # Replace multiple underscores with a single underscore + sanitized_name = re.sub(r'__+', '_', sanitized_name) + logger.debug('sanitized name: %s', sanitized_name) + if path: + target_dirs = [d.name for d in list(path.glob("*")) if d.is_dir()] + logger.debug('target dirs: %s', target_dirs) + new_name = sanitized_name + suf = 1 + while new_name in target_dirs: + suf = suf + 1 + new_name = f'{sanitized_name}_{suf}' + logger.debug('looping suffix: %s', new_name) + + sanitized_name = new_name + + return sanitized_name + + +def clean_object_id(name: str) -> str: + """Replace '/' and '+' with '/' in XCA object identifiers""" + splits = name.split('-x') + if len(splits) > 1: + return f"{splits[0]}-x{splits[1].replace('+', '/').replace('_', '/')}" + else: + return name.replace('+', '/').replace('_', '/') diff --git a/viewer/views.py b/viewer/views.py index 523ab9d9..71085d32 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3,11 +3,9 @@ import os import shlex import shutil -import uuid -import zipfile from datetime import datetime from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from wsgiref.util import FileWrapper import pandas as pd @@ -16,25 +14,21 @@ from celery.result import AsyncResult from dateutil.parser import parse from django.conf import settings -from django.core.files.base import ContentFile -from django.core.files.storage import default_storage -from django.core.mail import send_mail -from django.db import connections from django.http import FileResponse, HttpResponse, JsonResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.views import View -from rest_framework import permissions, status, viewsets -from rest_framework.exceptions import ParseError +from rest_framework import generics, mixins, permissions, status, viewsets from rest_framework.parsers import BaseParser from rest_framework.response import Response from rest_framework.views import APIView from api.infections import INFECTION_STRUCTURE_DOWNLOAD, have_infection -from api.security import ISpyBSafeQuerySet -from api.utils import get_highlighted_diffs, get_params, pretty_request +from api.security import ISPyBSafeQuerySet +from api.utils import get_highlighted_diffs, get_img_from_smiles, pretty_request +from service_status.models import Service from viewer import filters, models, serializers -from viewer.services import get_service_state +from viewer.permissions import IsObjectProposalMember from viewer.squonk2_agent import ( AccessParams, CommonParams, @@ -44,7 +38,13 @@ Squonk2AgentRv, get_squonk2_agent, ) -from viewer.utils import create_squonk_job_request_url, handle_uploaded_file +from viewer.utils import ( + CSV_TO_DICT_DOWNLOAD_ROOT, + create_csv_from_dict, + create_squonk_job_request_url, + handle_uploaded_file, + save_tmp_file, +) from .discourse import ( check_discourse_user, @@ -66,7 +66,6 @@ erase_compound_set_job_material, process_compound_set, process_compound_set_job_file, - process_design_sets, process_job_file_transfer, task_load_target, validate_compound_set, @@ -83,21 +82,168 @@ _SQ2A: Squonk2Agent = get_squonk2_agent() +_ISPYB_SAFE_QUERY_SET = ISPyBSafeQuerySet() + +# --------------------------- +# ENTRYPOINT FOR THE FRONTEND +# --------------------------- + + +def react(request): + """ + The f/e starts here. + This is the first API call that the front-end calls, and it returns a 'context' + defining the state of things like the legacy URL, Squonk and Discourse availability + via the 'context' variable used for the view's template. + """ + + # ---- + # NOTE: If you add or remove any context keys here + # ---- you MUST update the template that gets rendered + # (viewer/react_temp.html) so that is matches + # the keys you are creating and passing here. + + # Start building the context that will be passed to the template + context = {'legacy_url': settings.LEGACY_URL} + + # Is the Squonk2 Agent configured? + logger.info("Checking whether Squonk2 is configured...") + sq2_rv = _SQ2A.configured() + if sq2_rv.success: + logger.info("Squonk2 is configured") + context['squonk_available'] = 'true' + else: + logger.info("Squonk2 is NOT configured") + context['squonk_available'] = 'false' + + discourse_api_key = settings.DISCOURSE_API_KEY + context['discourse_available'] = 'true' if discourse_api_key else 'false' + user = request.user + if user.is_authenticated: + context['discourse_host'] = '' + context['user_present_on_discourse'] = 'false' + # If user is authenticated and a discourse api key is available, + # hen check discourse to see if user is set up and set up flag in context. + if discourse_api_key: + context['discourse_host'] = settings.DISCOURSE_HOST + _, _, user_id = check_discourse_user(user) + if user_id: + context['user_present_on_discourse'] = 'true' + + # User is authenticated, so if Squonk can be called + # return the Squonk UI URL + # so the f/e knows where to go to find it. + context['squonk_ui_url'] = '' + if sq2_rv.success and check_squonk_active(request): + context['squonk_ui_url'] = _SQ2A.get_ui_url() + + context['target_warning_message'] = settings.TARGET_WARNING_MESSAGE + + render_template = "viewer/react_temp.html" + logger.info("Rendering %s with context=%s...", render_template, context) + return render(request, render_template, context) + + +# -------------------- +# FUNCTION-BASED VIEWS +# -------------------- + + +def img_from_smiles(request): + """Generate a 2D molecule image for a given smiles string""" + if "smiles" in request.GET and (smiles := request.GET["smiles"]): + return get_img_from_smiles(smiles, request) + else: + return HttpResponse("Please insert SMILES") + + +def highlight_mol_diff(request): + """Generate a 2D molecule image highlighting the difference between a + reference and new molecule + """ + if 'ref_smiles' in request.GET: + return HttpResponse(get_highlighted_diffs(request)) + else: + return HttpResponse("Please insert smiles for reference and probe") + + +def get_open_targets(request): + """Return a list of all open targets (viewer/open_targets)""" + # Unused arguments + del request + + targets = models.Target.objects.all() + target_names = [] + target_ids = [] + + open_proposals: set = _ISPYB_SAFE_QUERY_SET.get_open_proposals() + for t in targets: + for p in t.project_id.all(): + if p.title in open_proposals: + target_names.append(t.title) + target_ids.append(t.id) + break + + return HttpResponse( + json.dumps({'target_names': target_names, 'target_ids': target_ids}) + ) + + +def computed_set_download(request, name): + """View to download an SDF file of a ComputedSet by name + (viewer/compound_set/()). + """ + # Unused arguments + del request + + computed_set = models.ComputedSet.objects.get(name=name) + if not computed_set: + return HttpResponse(status=status.HTTP_404_NOT_FOUND) + # Is the computed set available to the user? + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + request.user, computed_set.target + ): + return HttpResponse( + "You are not a member of the CompoundSet's Target Proposal", + status=status.HTTP_403_FORBIDDEN, + ) + + written_filename = computed_set.written_sdf_filename + with open(written_filename, 'r', encoding='utf-8') as wf: + data = wf.read() + response = HttpResponse(content_type='text/plain') + response[ + 'Content-Disposition' + ] = f'attachment; filename={name}.sdf' # force browser to download file + response.write(data) + return response + + +# ----------------- +# CLASS-BASED VIEWS +# ----------------- -class CompoundIdentifierTypeView(viewsets.ModelViewSet): + +class CompoundIdentifierTypeView(viewsets.ReadOnlyModelViewSet): queryset = models.CompoundIdentifierType.objects.all() serializer_class = serializers.CompoundIdentifierTypeSerializer permission_classes = [permissions.IsAuthenticated] -class CompoundIdentifierView(viewsets.ModelViewSet): +class CompoundIdentifierView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): queryset = models.CompoundIdentifier.objects.all() serializer_class = serializers.CompoundIdentifierSerializer - permission_classes = [permissions.IsAuthenticated] + filter_permissions = "compound__project_id" + permission_classes = [IsObjectProposalMember] filterset_fields = ["type", "compound"] -class VectorsView(ISpyBSafeQuerySet): +class VectorsView(ISPyBSafeQuerySet): """Vectors (api/vector)""" queryset = models.SiteObservation.filter_manager.filter_qs() @@ -106,7 +252,7 @@ class VectorsView(ISpyBSafeQuerySet): filter_permissions = "experiment__experiment_upload__target__project_id" -class MolecularPropertiesView(ISpyBSafeQuerySet): +class MolecularPropertiesView(ISPyBSafeQuerySet): """Molecular properties (api/molprops)""" queryset = models.Compound.filter_manager.filter_qs() @@ -115,7 +261,7 @@ class MolecularPropertiesView(ISpyBSafeQuerySet): filter_permissions = "experiment__experiment_upload__target__project_id" -class GraphView(ISpyBSafeQuerySet): +class GraphView(ISPyBSafeQuerySet): """Graph (api/graph)""" queryset = models.SiteObservation.filter_manager.filter_qs() @@ -124,7 +270,7 @@ class GraphView(ISpyBSafeQuerySet): filter_permissions = "experiment__experiment_upload__target__project_id" -class MolImageView(ISpyBSafeQuerySet): +class MolImageView(ISPyBSafeQuerySet): """Molecule images (api/molimg)""" queryset = models.SiteObservation.objects.filter() @@ -133,7 +279,7 @@ class MolImageView(ISpyBSafeQuerySet): filter_permissions = "experiment__experiment_upload__target__project_id" -class CompoundImageView(ISpyBSafeQuerySet): +class CompoundImageView(ISPyBSafeQuerySet): """Compound images (api/cmpdimg)""" queryset = models.Compound.filter_manager.filter_qs() @@ -142,7 +288,7 @@ class CompoundImageView(ISpyBSafeQuerySet): filterset_class = filters.CmpdImgFilter -class ProteinMapInfoView(ISpyBSafeQuerySet): +class ProteinMapInfoView(ISPyBSafeQuerySet): """Protein map info (file) (api/protmap)""" queryset = models.SiteObservation.objects.all() @@ -155,7 +301,7 @@ class ProteinMapInfoView(ISpyBSafeQuerySet): ) -class ProteinPDBInfoView(ISpyBSafeQuerySet): +class ProteinPDBInfoView(ISPyBSafeQuerySet): """Protein apo pdb info (file) (api/protpdb)""" queryset = models.SiteObservation.objects.all() @@ -168,7 +314,7 @@ class ProteinPDBInfoView(ISpyBSafeQuerySet): ) -class ProteinPDBBoundInfoView(ISpyBSafeQuerySet): +class ProteinPDBBoundInfoView(ISPyBSafeQuerySet): """Protein bound pdb info (file) (api/protpdbbound)""" queryset = models.SiteObservation.filter_manager.filter_qs() @@ -181,7 +327,7 @@ class ProteinPDBBoundInfoView(ISpyBSafeQuerySet): ) -class ProjectView(ISpyBSafeQuerySet): +class ProjectView(ISPyBSafeQuerySet): """Projects (api/project)""" queryset = models.Project.objects.filter() @@ -190,36 +336,35 @@ class ProjectView(ISpyBSafeQuerySet): filter_permissions = "" -class TargetView(ISpyBSafeQuerySet): - """Targets (api/targets)""" - +class TargetView(mixins.UpdateModelMixin, ISPyBSafeQuerySet): queryset = models.Target.objects.filter() serializer_class = serializers.TargetSerializer filter_permissions = "project_id" filterset_fields = ("title",) + permission_classes = [IsObjectProposalMember] def patch(self, request, pk): try: target = self.queryset.get(pk=pk) except models.Target.DoesNotExist: + msg = f"Target pk={pk} does not exist" + logger.warning(msg) return Response( - {"message": f"Target pk={pk} does not exist"}, + {"message": msg}, status=status.HTTP_404_NOT_FOUND, ) serializer = self.serializer_class(target, data=request.data, partial=True) if serializer.is_valid(): - logger.debug("serializer data: %s", serializer.validated_data) - serializer.save() + _ = serializer.save() return Response(serializer.data, status=status.HTTP_200_OK) else: - logger.debug("serializer error: %s", serializer.errors) return Response( {"message": "wrong parameters"}, status=status.HTTP_400_BAD_REQUEST ) -class CompoundView(ISpyBSafeQuerySet): +class CompoundView(ISPyBSafeQuerySet): """Compounds (api/compounds)""" queryset = models.Compound.filter_manager.filter_qs() @@ -228,117 +373,14 @@ class CompoundView(ISpyBSafeQuerySet): filterset_class = filters.CompoundFilter -def react(request): - """We "START HERE". This is the first API call that the front-end calls.""" - - discourse_api_key = settings.DISCOURSE_API_KEY - - context = {} - - # Legacy URL (a n optional prior stack) - # May be blank ('') - context['legacy_url'] = settings.LEGACY_URL - - # Is the Squonk2 Agent configured? - logger.info("Checking whether Squonk2 is configured...") - sq2_rv = _SQ2A.configured() - if sq2_rv.success: - logger.info("Squonk2 is configured") - context['squonk_available'] = 'true' - else: - logger.info("Squonk2 is NOT configured") - context['squonk_available'] = 'false' - - if discourse_api_key: - context['discourse_available'] = 'true' - else: - context['discourse_available'] = 'false' - - user = request.user - if user.is_authenticated: - context['discourse_host'] = '' - context['user_present_on_discourse'] = 'false' - # If user is authenticated and a discourse api key is available, then check discourse to - # see if user is set up and set up flag in context. - if discourse_api_key: - context['discourse_host'] = settings.DISCOURSE_HOST - _, _, user_id = check_discourse_user(user) - if user_id: - context['user_present_on_discourse'] = 'true' - - # If user is authenticated Squonk can be called then return the Squonk host - # so the Frontend can navigate to it - context['squonk_ui_url'] = '' - if sq2_rv.success and check_squonk_active(request): - context['squonk_ui_url'] = _SQ2A.get_ui_url() - - return render(request, "viewer/react_temp.html", context) - - -def save_pdb_zip(pdb_file): - zf = zipfile.ZipFile(pdb_file) - zip_lst = zf.namelist() - zfile = {} - zfile_hashvals: Dict[str, str] = {} - print(zip_lst) - for filename in zip_lst: - # only handle pdb files - if filename.split('.')[-1] == 'pdb': - f = filename.split('/')[0] - save_path = os.path.join(settings.MEDIA_ROOT, 'tmp', f) - if default_storage.exists(f): - rand_str = uuid.uuid4().hex - pdb_path = default_storage.save( - save_path.replace('.pdb', f'-{rand_str}.pdb'), - ContentFile(zf.read(filename)), - ) - # Test if Protein object already exists - # code = filename.split('/')[-1].replace('.pdb', '') - # test_pdb_code = filename.split('/')[-1].replace('.pdb', '') - # test_prot_objs = Protein.objects.filter(code=test_pdb_code) - # - # if len(test_prot_objs) > 0: - # # make a unique pdb code as not to overwrite existing object - # rand_str = uuid.uuid4().hex - # test_pdb_code = f'{code}#{rand_str}' - # zfile_hashvals[code] = rand_str - # - # fn = test_pdb_code + '.pdb' - # - # pdb_path = default_storage.save('tmp/' + fn, - # ContentFile(zf.read(filename))) - else: - pdb_path = default_storage.save( - save_path, ContentFile(zf.read(filename)) - ) - test_pdb_code = pdb_path.split('/')[-1].replace('.pdb', '') - zfile[test_pdb_code] = pdb_path - - # Close the zip file - if zf: - zf.close() - - return zfile, zfile_hashvals - - -def save_tmp_file(myfile): - """Save file in temporary location for validation/upload processing""" - - name = myfile.name - path = default_storage.save('tmp/' + name, ContentFile(myfile.read())) - tmp_file = str(os.path.join(settings.MEDIA_ROOT, path)) - - return tmp_file - - -class UploadCSet(APIView): - """Render and control viewer/upload-cset.html - a page allowing upload of computed sets. Validation and +class UploadComputedSetView(generics.ListCreateAPIView): + """Render and control viewer/upload-cset.html - a page allowing upload of computed sets. Validation and upload tasks are defined in `viewer.compound_set_upload`, `viewer.sdf_check` and `viewer.tasks` and the task response handling is done by `viewer.views.ValidateTaskView` and `viewer.views.UploadTaskView` """ def get(self, request): - tag = '+ UploadCSet GET' + tag = '+ UploadComputedSetView GET' logger.info('%s', pretty_request(request, tag=tag)) logger.info('User=%s', str(request.user)) # logger.info('Auth=%s', str(request.auth)) @@ -375,7 +417,7 @@ def get(self, request): return render(request, 'viewer/upload-cset.html', context) def post(self, request): - tag = '+ UploadCSet POST' + tag = '+ UploadComputedSetView POST' logger.info('%s', pretty_request(request, tag=tag)) logger.info('User=%s', str(request.user)) # logger.info('Auth=%s', str(request.auth)) @@ -393,7 +435,7 @@ def post(self, request): user = self.request.user logger.info( - '+ UploadCSet POST user.id=%s choice="%s" target="%s" update_set="%s"', + '+ UploadComputedSetView POST user.id=%s choice="%s" target="%s" update_set="%s"', user.id, choice, target, @@ -410,7 +452,7 @@ def post(self, request): else: request.session[_SESSION_ERROR] = 'The set could not be found' logger.warning( - '- UploadCSet POST error_msg="%s"', + '- UploadComputedSetView POST error_msg="%s"', request.session[_SESSION_ERROR], ) return redirect('viewer:upload_cset') @@ -424,7 +466,7 @@ def post(self, request): ' you must provide a Target and SDF file' ) logger.warning( - '- UploadCSet POST error_msg="%s"', + '- UploadComputedSetView POST error_msg="%s"', request.session[_SESSION_ERROR], ) return redirect('viewer:upload_cset') @@ -434,7 +476,7 @@ def post(self, request): _SESSION_ERROR ] = 'To Delete you must select an existing set' logger.warning( - '- UploadCSet POST error_msg="%s"', + '- UploadComputedSetView POST error_msg="%s"', request.session[_SESSION_ERROR], ) return redirect('viewer:upload_cset') @@ -455,11 +497,10 @@ def post(self, request): # If so redirect... if _SESSION_ERROR in request.session: logger.warning( - '- UploadCSet POST error_msg="%s"', + '- UploadComputedSetView POST error_msg="%s"', request.session[_SESSION_ERROR], ) return redirect('viewer:upload_cset') - # You cannot validate or upload a set # unless the user is part of the Target's project (proposal) # even if the target is 'open'. @@ -472,15 +513,9 @@ def post(self, request): context['error_message'] = f'Unknown Target ({target})' return render(request, 'viewer/upload-cset.html', context) # What proposals is the user a member of? - ispyb_safe_query_set = ISpyBSafeQuerySet() - user_proposals = ispyb_safe_query_set.get_proposals_for_user( - user, restrict_to_membership=True - ) - user_is_member = any( - target_project.title in user_proposals - for target_project in target_record.project_id.all() - ) - if not user_is_member: + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + user, target_record + ): context[ 'error_message' ] = f"You cannot load compound sets for '{target}'. You are not a member of any of its Proposals" @@ -521,7 +556,7 @@ def post(self, request): task_id = task_validate.id task_status = task_validate.status logger.info( - '+ UploadCSet POST "Validate" task underway' + '+ UploadComputedSetView POST "Validate" task underway' ' (validate_task_id=%s (%s) validate_task_status=%s)', task_id, type(task_id), @@ -555,7 +590,7 @@ def post(self, request): task_id = task_upload.id task_status = task_upload.status logger.info( - '+ UploadCSet POST "Upload" task underway' + '+ UploadComputedSetView POST "Upload" task underway' ' (upload_task_id=%s (%s) upload_task_status=%s)', task_id, type(task_id), @@ -571,7 +606,34 @@ def post(self, request): assert selected_set written_sdf_filename = selected_set.written_sdf_filename selected_set_name = selected_set.name + + # related objects: + # - ComputedSetComputedMolecule + # - ComputedMolecule + # - NumericalScoreValues + # - TextScoreValues + # - ComputedMolecule_computed_inspirations + # - Compound + + # all but ComputedMolecule are handled automatically + # but (because of the m2m), have to delete those + # separately + + # select ComputedMolecule objects that are in this set + # and not in any other sets + # fmt: off + selected_set.computed_molecules.exclude( + pk__in=models.ComputedMolecule.objects.filter( + computed_set__in=models.ComputedSet.objects.filter( + target=selected_set.target, + ).exclude( + pk=selected_set.pk, + ), + ), + ).delete() + # fmt: on selected_set.delete() + # ...and the original (expected) file if os.path.isfile(written_sdf_filename): os.remove(written_sdf_filename) @@ -580,70 +642,27 @@ def post(self, request): _SESSION_MESSAGE ] = f'Compound set "{selected_set_name}" deleted' - logger.info('+ UploadCSet POST "Delete" done') + logger.info('+ UploadComputedSetView POST "Delete" done') return redirect('viewer:upload_cset') else: logger.warning( - '+ UploadCSet POST unsupported submit_choice value (%s)', choice + '+ UploadComputedSetView POST unsupported submit_choice value (%s)', + choice, ) else: - logger.warning('- UploadCSet POST form.is_valid() returned False') + logger.warning( + '- UploadComputedSetView POST form.is_valid() returned False' + ) - logger.info('- UploadCSet POST (leaving)') + logger.info('- UploadComputedSetView POST (leaving)') context = {'form': form} return render(request, 'viewer/upload-cset.html', context) -def email_task_completion( - contact_email, message_type, target_name, target_path=None, task_id=None -): - """Notify user of upload completion""" - - logger.info('+ email_notify_task_completion: ' + message_type + ' ' + target_name) - email_from = settings.EMAIL_HOST_USER - - if contact_email == '' or not email_from: - # Only send email if configured. - return - - if message_type == 'upload-success': - subject = 'Fragalysis: Target: ' + target_name + ' Uploaded' - message = ( - 'The upload of your target data is complete. Your target is available at: ' - + str(target_path) - ) - elif message_type == 'validate-success': - subject = 'Fragalysis: Target: ' + target_name + ' Validation' - message = ( - 'Your data was validated. It can now be uploaded using the upload option.' - ) - else: - # Validation failure - subject = 'Fragalysis: Target: ' + target_name + ' Validation/Upload Failed' - message = ( - 'The validation/upload of your target data did not complete successfully. ' - 'Please navigate the following link to check the errors: validate_task/' - + str(task_id) - ) - - recipient_list = [ - contact_email, - ] - logger.info('+ email_notify_task_completion email_from: %s', email_from) - logger.info('+ email_notify_task_completion subject: %s', subject) - logger.info('+ email_notify_task_completion message: %s', message) - logger.info('+ email_notify_task_completion contact_email: %s', contact_email) - - # Send email - this should not prevent returning to the screen in the case of error. - send_mail(subject, message, email_from, recipient_list, fail_silently=True) - logger.info('- email_notify_task_completion') - return - - class ValidateTaskView(View): """View to handle dynamic loading of validation results from `viewer.tasks.validate`. The validation of files uploaded to viewer/upload_cset. @@ -670,9 +689,6 @@ def get(self, request, validate_task_id): - html (str): html of task outcome - success message or html table of errors & fail message """ - # Unused arguments - del request - logger.info('+ ValidateTaskView.get') validate_task_id_str = str(validate_task_id) @@ -696,6 +712,28 @@ def get(self, request, validate_task_id): # Response from validation is a tuple validate_dict = results[1] validated = results[2] + # [3] comes from task in tasks.py, 4th element in task payload tuple + task_data = results[3] + + if isinstance(task_data, dict) and 'target' in task_data.keys(): + target_name = task_data['target'] + try: + target = models.Target.objects.get(title=target_name) + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + request.user, target + ): + return Response( + {'error': "You are not a member of the Target's proposal"}, + status=status.HTTP_403_FORBIDDEN, + ) + except models.Target.DoesNotExist: + # the name is filled from db, so this not existing would be extraordinary + logger.error('Target %s not found', target_name) + return Response( + {'error': f'Target {target_name} not found'}, + status=status.HTTP_403_FORBIDDEN, + ) + if validated: response_data[ 'html' @@ -778,9 +816,6 @@ def get(self, request, upload_task_id): - processed (str): 'None' - html (str): message to tell the user their data was not processed """ - # Unused arguments - del request - logger.debug('+ UploadTaskView.get') upload_task_id_str = str(upload_task_id) task = AsyncResult(upload_task_id_str) @@ -822,6 +857,15 @@ def get(self, request, upload_task_id): response_data['validated'] = 'Validated' cset_name = results[1] cset = models.ComputedSet.objects.get(name=cset_name) + + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + request.user, cset.target + ): + return Response( + {'error': "You are not a member of the Target's proposal"}, + status=status.HTTP_403_FORBIDDEN, + ) + name = cset.name response_data['results'] = {} response_data['results']['cset_download_url'] = ( @@ -843,125 +887,6 @@ def get(self, request, upload_task_id): return JsonResponse(response_data) -def img_from_smiles(request): - """Generate a 2D molecule image for a given smiles string""" - if "smiles" in request.GET: - smiles = request.GET["smiles"] - if smiles: - return get_params(smiles, request) - else: - return HttpResponse("Please insert SMILES") - else: - return HttpResponse("Please insert SMILES") - - -def highlight_mol_diff(request): - """Generate a 2D molecule image highlighting the difference between a - reference and new molecule - """ - if 'prb_smiles' and 'ref_smiles' in request.GET: - return HttpResponse(get_highlighted_diffs(request)) - else: - return HttpResponse("Please insert smiles for reference and probe") - - -def similarity_search(request): - if "smiles" in request.GET: - smiles = request.GET["smiles"] - else: - return HttpResponse("Please insert SMILES") - if "db_name" in request.GET: - db_name = request.GET["db_name"] - else: - return HttpResponse("Please insert db_name") - sql_query = """SELECT sub.* - FROM ( - SELECT rdk.id,rdk.structure,rdk.idnumber - FROM vendordbs.enamine_real_dsi_molfps AS mfp - JOIN vendordbs.enamine_real_dsi AS rdk ON mfp.id = rdk.id - WHERE m @> qmol_from_smiles(%s) LIMIT 1000 - ) sub;""" - with connections[db_name].cursor() as cursor: - cursor.execute(sql_query, [smiles]) - return HttpResponse(json.dumps(cursor.fetchall())) - - -def get_open_targets(request): - """Return a list of all open targets (viewer/open_targets)""" - # Unused arguments - del request - - targets = models.Target.objects.all() - target_names = [] - target_ids = [] - - for t in targets: - for p in t.project_id.all(): - if 'OPEN' in p.title: - target_names.append(t.title) - target_ids.append(t.id) - - return HttpResponse( - json.dumps({'target_names': target_names, 'target_ids': target_ids}) - ) - - -def cset_download(request, name): - """View to download an SDF file of a ComputedSet by name - (viewer/compound_set/()). - """ - # Unused arguments - del request - - computed_set = models.ComputedSet.objects.get(name=name) - written_filename = computed_set.written_sdf_filename - with open(written_filename, 'r', encoding='utf-8') as wf: - data = wf.read() - response = HttpResponse(content_type='text/plain') - response[ - 'Content-Disposition' - ] = f'attachment; filename={name}.sdf' # force browser to download file - response.write(data) - return response - - -def pset_download(request, name): - """View to download a zip file of all protein structures (apo) for a computed set - (viewer/compound_set/()) - """ - # Unused arguments - del request - - response = HttpResponse(content_type='application/zip') - filename = 'protein-set_' + name + '.zip' - response['Content-Disposition'] = ( - 'filename=%s' % filename - ) # force browser to download file - - # For the first stage (green release) of the XCA-based Fragalysis Stack - # there are no PDB files. - # compound_set = models.ComputedSet.objects.get(name=name) - # computed_molecules = models.ComputedMolecule.objects.filter(computed_set=compound_set) - # pdb_filepaths = list(set([c.pdb_info.path for c in computed_molecules])) - # buff = StringIO() - # zip_obj = zipfile.ZipFile(buff, 'w') - # zip_obj.writestr('') - # for fp in pdb_filepaths: - # data = open(fp, 'r', encoding='utf-8').read() - # zip_obj.writestr(fp.split('/')[-1], data) - # zip_obj.close() - # buff.flush() - # ret_zip = buff.getvalue() - # buff.close() - - # ...instead we just create an empty file... - with zipfile.ZipFile('dummy.zip', 'w') as pdb_file: - pass - - response.write(pdb_file) - return response - - # Start of ActionType class ActionTypeView(viewsets.ModelViewSet): """View to retrieve information about action types available to users (GET). @@ -980,76 +905,79 @@ class ActionTypeView(viewsets.ModelViewSet): # Start of Session Project -class SessionProjectsView(viewsets.ModelViewSet): +class SessionProjectView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """View to retrieve information about user projects (collection of sessions) (GET). Also used for saving project information (PUT, POST, PATCH). (api/session-projects). """ queryset = models.SessionProject.objects.filter() + filter_permissions = "target__project_id" + filterset_fields = '__all__' def get_serializer_class(self): - """Determine which serializer to use based on whether the request is a GET or a POST, PUT or PATCH request - - Returns - ------- - Serializer (rest_framework.serializers.ModelSerializer): - - if GET: `viewer.serializers.SessionProjectReadSerializer` - - if other: `viewer.serializers.SessionProjectWriteSerializer` - """ if self.request.method in ['GET']: - # GET return serializers.SessionProjectReadSerializer - # (POST, PUT, PATCH) return serializers.SessionProjectWriteSerializer - filter_permissions = "target_id__project_id" - filterset_fields = '__all__' - -class SessionActionsView(viewsets.ModelViewSet): +class SessionActionsView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """View to retrieve information about actions relating to sessions_project (GET). Also used for saving project action information (PUT, POST, PATCH). (api/session-actions). """ - queryset = models.SessionActions.objects.filter() + queryset = models.SessionActions.filter_manager.filter_qs() + filter_permissions = "session_project__target__project_id" serializer_class = serializers.SessionActionsSerializer # Note: jsonField for Actions will need specific queries - can introduce if needed. filterset_fields = ('id', 'author', 'session_project', 'last_update_date') -class SnapshotsView(viewsets.ModelViewSet): +class SnapshotView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """View to retrieve information about user sessions (snapshots) (GET). Also used for saving session information (PUT, POST, PATCH). (api/snapshots) """ - queryset = models.Snapshot.objects.filter() + queryset = models.Snapshot.filter_manager.filter_qs() + filter_permissions = "session_project__target__project_id" + filterset_class = filters.SnapshotFilter def get_serializer_class(self): - """Determine which serializer to use based on whether the request is a GET or a POST, PUT or PATCH request - - Returns - ------- - Serializer (rest_framework.serializers.ModelSerializer): - - if GET: `viewer.serializers.SnapshotReadSerializer` - - if other: `viewer.serializers.SnapshotWriteSerializer` - """ if self.request.method in ['GET']: return serializers.SnapshotReadSerializer return serializers.SnapshotWriteSerializer - filterset_class = filters.SnapshotFilter - -class SnapshotActionsView(viewsets.ModelViewSet): +class SnapshotActionsView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """View to retrieve information about actions relating to snapshots (GET). Also used for saving snapshot action information (PUT, POST, PATCH). (api/snapshot-actions). """ - queryset = models.SnapshotActions.objects.filter() + queryset = models.SnapshotActions.filter_manager.filter_qs() + filter_permissions = "snapshot__session_project__target__project_id" serializer_class = serializers.SnapshotActionsSerializer # Note: jsonField for Actions will need specific queries - can introduce if needed. @@ -1062,7 +990,7 @@ class SnapshotActionsView(viewsets.ModelViewSet): ) -class DSetCSVParser(BaseParser): +class DesignSetCSVParser(BaseParser): """ CSV parser class specific to design set csv spec - sets media_type for DSetUploadView to text/csv @@ -1071,60 +999,72 @@ class DSetCSVParser(BaseParser): media_type = 'text/csv' -class DSetUploadView(APIView): +class DesignSetUploadView(APIView): """Upload a design set (PUT) from a csv file""" - parser_class = (DSetCSVParser,) + parser_class = (DesignSetCSVParser,) def put(self, request, format=None): # pylint: disable=redefined-builtin """Method to handle PUT request and upload a design set""" # Don't need... - del format - - f = request.FILES['file'] - set_type = request.PUT['type'] - set_description = request.PUT['description'] - - # save uploaded file to temporary storage - name = f.name - path = default_storage.save('tmp/' + name, ContentFile(f.read())) - tmp_file = str(os.path.join(settings.MEDIA_ROOT, path)) - - df = pd.read_csv(tmp_file) - mandatory_cols = ['set_name', 'smiles', 'identifier', 'inspirations'] - actual_cols = df.columns - for col in mandatory_cols: - if col not in actual_cols: - raise ParseError( - "The 4 following columns are mandatory: set_name, smiles, identifier, inspirations" - ) + del format, request - set_names, compounds = process_design_sets(df, set_type, set_description) + # Unsupported for now, as part of 1247 (securing endpoints) + return HttpResponse(status=status.HTTP_404_NOT_FOUND) - string = 'Design set(s) successfully created: ' + # BEGIN removed as part of 1247 (securing endpoints) + # This code is unused by the f/e - length = len(set_names) - string += str(length) + '; ' - for i in range(0, length): - string += ( - str(i + 1) - + ' - ' - + set_names[i] - + ') number of compounds = ' - + str(len(compounds[i])) - + '; ' - ) + # f = request.FILES['file'] + # set_type = request.PUT['type'] + # set_description = request.PUT['description'] - return HttpResponse(json.dumps(string)) + # # save uploaded file to temporary storage + # name = f.name + # path = default_storage.save('tmp/' + name, ContentFile(f.read())) + # tmp_file = str(os.path.join(settings.MEDIA_ROOT, path)) + # df = pd.read_csv(tmp_file) + # mandatory_cols = ['set_name', 'smiles', 'identifier', 'inspirations'] + # actual_cols = df.columns + # for col in mandatory_cols: + # if col not in actual_cols: + # raise ParseError( + # "The 4 following columns are mandatory: set_name, smiles, identifier, inspirations" + # ) -class ComputedSetView(viewsets.ModelViewSet): + # set_names, compounds = process_design_sets(df, set_type, set_description) + + # string = 'Design set(s) successfully created: ' + + # length = len(set_names) + # string += str(length) + '; ' + # for i in range(0, length): + # string += ( + # str(i + 1) + # + ' - ' + # + set_names[i] + # + ') number of compounds = ' + # + str(len(compounds[i])) + # + '; ' + # ) + + # return HttpResponse(json.dumps(string)) + + # END removed as part of 1247 (securing endpoints) + + +class ComputedSetView( + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """Retrieve information about and delete computed sets.""" queryset = models.ComputedSet.objects.filter() serializer_class = serializers.ComputedSetSerializer - filter_permissions = "project_id" + filter_permissions = "target__project_id" filterset_fields = ('target', 'target__title') + permission_classes = [IsObjectProposalMember] http_method_names = ['get', 'head', 'delete'] @@ -1140,52 +1080,52 @@ def destroy(self, request, pk=None): return HttpResponse(status=204) -class ComputedMoleculesView(viewsets.ReadOnlyModelViewSet): +class ComputedMoleculesView(ISPyBSafeQuerySet): """Retrieve information about computed molecules - 3D info (api/compound-molecules).""" queryset = models.ComputedMolecule.objects.all() serializer_class = serializers.ComputedMoleculeSerializer - filter_permissions = "project_id" + filter_permissions = "compound__project_id" filterset_fields = ('computed_set',) -class NumericalScoresView(viewsets.ReadOnlyModelViewSet): +class NumericalScoreValuesView(ISPyBSafeQuerySet): """View to retrieve information about numerical computed molecule scores (api/numerical-scores). """ queryset = models.NumericalScoreValues.objects.all() serializer_class = serializers.NumericalScoreSerializer - filter_permissions = "project_id" + filter_permissions = "compound__compound__project_id" filterset_fields = ('compound', 'score') -class TextScoresView(viewsets.ReadOnlyModelViewSet): +class TextScoresView(ISPyBSafeQuerySet): """View to retrieve information about text computed molecule scores (api/text-scores).""" queryset = models.TextScoreValues.objects.all() serializer_class = serializers.TextScoreSerializer - filter_permissions = "project_id" + filter_permissions = "compound__compound__project_id" filterset_fields = ('compound', 'score') -class CompoundScoresView(viewsets.ReadOnlyModelViewSet): +class CompoundScoresView(ISPyBSafeQuerySet): """View to retrieve descriptions of scores for a given name or computed set.""" queryset = models.ScoreDescription.objects.all() serializer_class = serializers.ScoreDescriptionSerializer - filter_permissions = "project_id" + filter_permissions = "computed_set__target__project_id" filterset_fields = ('computed_set', 'name') -class ComputedMolAndScoreView(viewsets.ReadOnlyModelViewSet): +class ComputedMolAndScoreView(ISPyBSafeQuerySet): """View to retrieve all information about molecules from a computed set along with all of their scores. """ queryset = models.ComputedMolecule.objects.all() serializer_class = serializers.ComputedMolAndScoreSerializer - filter_permissions = "project_id" + filter_permissions = "compound__project_id" filterset_fields = ('computed_set',) @@ -1217,6 +1157,12 @@ class DiscoursePostView(viewsets.ViewSet): def create(self, request): """Method to handle POST request and call discourse to create the post""" logger.info('+ DiscoursePostView.post') + if not request.user.is_authenticated: + content: Dict[str, Any] = { + 'error': 'Only authenticated users can post content to Discourse' + } + return Response(content, status=status.HTTP_403_FORBIDDEN) + data = request.data logger.info('+ DiscoursePostView.post %s', json.dumps(data)) @@ -1273,71 +1219,70 @@ def list(self, request): return Response({"Posts": posts}) -def create_csv_from_dict(input_dict, title=None, filename=None): - """Write a CSV file containing data from an input dictionary and return a URL - to the file in the media directory. - """ - if not filename: - filename = 'download' - - media_root = settings.MEDIA_ROOT - unique_dir = str(uuid.uuid4()) - # /code/media/downloads/unique_dir - download_path = os.path.join(media_root, 'downloads', unique_dir) - os.makedirs(download_path, exist_ok=True) - - download_file = os.path.join(download_path, filename) - - # Remove file if it already exists - if os.path.isfile(download_file): - os.remove(download_file) - - with open(download_file, "w", newline='', encoding='utf-8') as csvfile: - if title: - csvfile.write(title) - csvfile.write("\n") - - df = pd.DataFrame.from_dict(input_dict) - df.to_csv(download_file, mode='a', header=True, index=False) - - return download_file - - -class DictToCsv(viewsets.ViewSet): +class DictToCSVView(viewsets.ViewSet): """Takes a dictionary and returns a download link to a CSV file with the data.""" serializer_class = serializers.DictToCsvSerializer def list(self, request): - """Method to handle GET request""" - file_url = request.GET.get('file_url') - - if file_url and os.path.isfile(file_url): - with open(file_url, encoding='utf8') as csvfile: - # return file and tidy up. - response = HttpResponse(csvfile, content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename=download.csv' - shutil.rmtree(os.path.dirname(file_url), ignore_errors=True) - return response - else: - return Response("Please provide file_url parameter") + """Method to handle GET request. + If the file exists it is returned and then removed.""" + file_url = request.GET.get('file_url', '') + logger.info('DictToCsv file_url="%s"', file_url) + + # The file is expected to include a full path + # to a file in the dicttocsv directory. + real_file_url = os.path.realpath(file_url) + if ( + os.path.commonpath([CSV_TO_DICT_DOWNLOAD_ROOT, real_file_url]) + != CSV_TO_DICT_DOWNLOAD_ROOT + ): + logger.warning( + 'DictToCsv path is invalid (file_url="%s" real_file_url="%s")', + file_url, + real_file_url, + ) + return Response("Please provide a file_url for an existing DictToCsv file") + elif not os.path.isfile(real_file_url): + logger.warning( + 'DictToCsv file does not exist (file_url="%s" real_file_url="%s")', + file_url, + real_file_url, + ) + return Response("The given DictToCsv file does not exist") + + with open(real_file_url, encoding='utf8') as csvfile: + # return file and tidy up. + response = HttpResponse(csvfile, content_type='text/csv') + # response['Content-Disposition'] = 'attachment; filename=download.csv' + filename = str(Path(real_file_url).name) + response['Content-Disposition'] = f'attachment; filename={filename}' + shutil.rmtree(os.path.dirname(real_file_url), ignore_errors=True) + return response def create(self, request): - """Method to handle POST request""" - logger.info('+ DictToCsv.post') + """Method to handle POST request. Creates a file that the user + is then expected to GET.""" input_dict = request.data['dict'] input_title = request.data['title'] + filename = request.data.get('filename', 'download.csv') + logger.info('title="%s" input_dict size=%s', input_title, len(input_dict)) if not input_dict: - return Response({"message": "Please enter Dictionary"}) + return Response({"message": "Please provide a dictionary"}) else: - filename_url = create_csv_from_dict(input_dict, input_title) + file_url = create_csv_from_dict(input_dict, input_title, filename=filename) + logger.info( + 'Created file_url="%s" (size=%s)', + file_url, + os.path.getsize(file_url), + ) - return Response({"file_url": filename_url}) + return Response({"file_url": file_url}) # Classes Relating to Tags -class TagCategoryView(viewsets.ModelViewSet): +class TagCategoryView(viewsets.ReadOnlyModelViewSet): """Set up and retrieve information about tag categories (api/tag_category).""" queryset = models.TagCategory.objects.all() @@ -1345,10 +1290,16 @@ class TagCategoryView(viewsets.ModelViewSet): filterset_fields = ('id', 'category') -class SiteObservationTagView(viewsets.ModelViewSet): +class SiteObservationTagView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """Set up/retrieve information about tags relating to Molecules (api/molecule_tag)""" queryset = models.SiteObservationTag.objects.all() + filter_permissions = "target__project_id" serializer_class = serializers.SiteObservationTagSerializer filterset_fields = ( 'id', @@ -1360,24 +1311,38 @@ class SiteObservationTagView(viewsets.ModelViewSet): ) -class PoseView(viewsets.ModelViewSet): +class PoseView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """Set up/retrieve information about Poses (api/poses)""" queryset = models.Pose.filter_manager.filter_qs() + filter_permissions = "compound__project_id" serializer_class = serializers.PoseSerializer filterset_class = filters.PoseFilter - http_method_names = ('get', 'post', 'patch') -class SessionProjectTagView(viewsets.ModelViewSet): +class SessionProjectTagView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """Set up/retrieve information about tags relating to Session Projects.""" queryset = models.SessionProjectTag.objects.all() + filter_permissions = "target__project_id" serializer_class = serializers.SessionProjectTagSerializer filterset_fields = ('id', 'tag', 'category', 'target', 'session_projects') -class DownloadStructures(ISpyBSafeQuerySet): +class DownloadStructuresView( + mixins.CreateModelMixin, + ISPyBSafeQuerySet, +): """Uses a selected subset of the target data (proteins and booleans with suggested files) and creates a Zip file with the contents. @@ -1470,6 +1435,16 @@ def create(self, request): return Response(content, status=status.HTTP_404_NOT_FOUND) logger.info('Found Target record %r', target) + # Is the user part of the target's proposal? + # (or is it a public target?) + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + request.user, target, restrict_public_to_membership=False + ): + msg = 'You have not been given access to this Target' + logger.warning(msg) + content = {'message': msg} + return Response(content, status=status.HTTP_403_FORBIDDEN) + proteins_list = [ p.strip() for p in request.data.get('proteins', '').split(',') if p ] @@ -1515,7 +1490,7 @@ def create(self, request): return Response({"file_url": filename_url}) -class UploadTargetExperiments(ISpyBSafeQuerySet): +class UploadExperimentUploadView(ISPyBSafeQuerySet): serializer_class = serializers.TargetExperimentWriteSerializer permission_class = [permissions.IsAuthenticated] http_method_names = ('post',) @@ -1543,7 +1518,7 @@ def create(self, request, *args, **kwargs): return redirect(settings.LOGIN_URL) else: if target_access_string not in self.get_proposals_for_user( - user, restrict_to_membership=True + user, restrict_public_to_membership=True ): return Response( { @@ -1583,16 +1558,22 @@ def create(self, request, *args, **kwargs): return Response({'task_status_url': url}, status=status.HTTP_202_ACCEPTED) -class TaskStatus(APIView): +class TaskStatusView(APIView): def get(self, request, task_id, *args, **kwargs): """Given a task_id (a string) we try to return the status of the task, trying to handle unknown tasks as best we can. """ # Unused arguments - del request, args, kwargs + del args, kwargs logger.debug("task_id=%s", task_id) + if not request.user.is_authenticated and settings.AUTHENTICATE_UPLOAD: + content: Dict[str, Any] = { + 'error': 'Only authenticated users can check the task status' + } + return Response(content, status=status.HTTP_403_FORBIDDEN) + # task_id is a UUID, but Celery expects a string task_id_str = str(task_id) result = None @@ -1611,9 +1592,26 @@ def get(self, request, task_id, *args, **kwargs): messages = [] if hasattr(result, 'info'): if isinstance(result.info, dict): + # check if user is allowed to view task info + proposal = result.info.get('proposal_ref', '') + + if proposal not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user( + request.user + ): + return Response( + {'error': 'You are not a member of the proposal f"proposal"'}, + status=status.HTTP_403_FORBIDDEN, + ) + messages = result.info.get('description', []) elif isinstance(result.info, list): - messages = result.info + # this path should never materialize + logger.error('result.info attribute list instead of dict') + return Response( + {'error': 'You are not a member of the proposal f"proposal"'}, + status=status.HTTP_403_FORBIDDEN, + ) + # messages = result.info started = result.state != 'PENDING' finished = result.ready() @@ -1634,7 +1632,7 @@ def get(self, request, task_id, *args, **kwargs): return JsonResponse(data) -class DownloadTargetExperiments(viewsets.ModelViewSet): +class DownloadExperimentUploadView(viewsets.ModelViewSet): serializer_class = serializers.TargetExperimentDownloadSerializer permission_class = [permissions.IsAuthenticated] http_method_names = ('post',) @@ -1646,17 +1644,64 @@ def create(self, request, *args, **kwargs): # Unused arguments del args, kwargs - logger.info("+ DownloadTargetExperiments.create called") + logger.info("+ DownloadExperimentUploadView.create called") serializer = self.get_serializer_class()(data=request.data) if serializer.is_valid(): - # project = serializer.validated_data['project'] - # target = serializer.validated_data['target'] - filename = serializer.validated_data['filename'] + # To permit a download the user must be a member of the target's proposal + # (or the proposal must be open) + project: models.Project = serializer.validated_data['project'] + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + request.user, [project.title], restrict_public_to_membership=False + ): + return Response( + {'error': "You have no access to the Project"}, + status=status.HTTP_403_FORBIDDEN, + ) + target: models.Target = serializer.validated_data['target'] + if project not in target.project_id.all(): + return Response( + {'error': "The Target is not part of the given Project"}, + status=status.HTTP_403_FORBIDDEN, + ) - # source_dir = Path(settings.MEDIA_ROOT).joinpath(TARGET_LOADER_DATA) - source_dir = Path(settings.MEDIA_ROOT).joinpath('tmp') - file_path = source_dir.joinpath(filename) + # Now we have to search for an ExperimentUpload that matches the Target + # and the filename combination. + filename = serializer.validated_data['filename'] + exp_uploads: List[ + models.ExperimentUpload + ] = models.ExperimentUpload.objects.filter( + target=target, + file=filename, + ) + if len(exp_uploads) > 1: + return Response( + { + 'error': "More than one ExperimentUpload matches your Target and Filename" + }, + status=status.HTTP_400_INTERNAL_SERVER_ERROR, + ) + elif len(exp_uploads) == 0: + return Response( + {'error': "No ExperimentUpload matches your Target and Filename"}, + status=status.HTTP_404_NOT_FOUND, + ) + # Use the only experiment upload found. + # We don't need the user's filename any more, + # it's embedded in the ExperimentUpload's file field. + exp_upload: models.ExperimentUpload = exp_uploads[0] + file_path = exp_upload.get_download_path() + logger.info( + "Found exp_upload=%s file_path=%s", + exp_upload, + file_path, + ) + if not file_path.exists(): + return Response( + {'error': f"TargetExperiment file '{filename}' not found"}, + status=status.HTTP_404_NOT_FOUND, + ) + # Finally, return the file wrapper = FileWrapper(open(file_path, 'rb')) response = FileResponse(wrapper, content_type='application/zip') response['Content-Disposition'] = ( @@ -1664,10 +1709,11 @@ def create(self, request, *args, **kwargs): ) response['Content-Length'] = os.path.getsize(file_path) return response + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) -class TargetExperimentUploads(ISpyBSafeQuerySet): +class ExperimentUploadView(ISPyBSafeQuerySet): queryset = models.ExperimentUpload.objects.all() serializer_class = serializers.TargetExperimentReadSerializer permission_class = [permissions.IsAuthenticated] @@ -1676,39 +1722,36 @@ class TargetExperimentUploads(ISpyBSafeQuerySet): http_method_names = ('get',) -class SiteObservations(viewsets.ModelViewSet): +class SiteObservationView(ISPyBSafeQuerySet): queryset = models.SiteObservation.filter_manager.filter_qs().filter( superseded=False ) serializer_class = serializers.SiteObservationReadSerializer - permission_class = [permissions.IsAuthenticated] filterset_class = filters.SiteObservationFilter - filter_permissions = "target__project_id" - http_method_names = ('get',) + filter_permissions = "experiment__experiment_upload__project" -class CanonSites(viewsets.ModelViewSet): +class CanonSiteView(ISPyBSafeQuerySet): queryset = models.CanonSite.filter_manager.filter_qs().filter(superseded=False) serializer_class = serializers.CanonSiteReadSerializer - permission_class = [permissions.IsAuthenticated] filterset_class = filters.CanonSiteFilter - http_method_names = ('get',) + filter_permissions = ( + "ref_conf_site__ref_site_observation__experiment__experiment_upload__project" + ) -class CanonSiteConfs(viewsets.ModelViewSet): +class CanonSiteConfView(ISPyBSafeQuerySet): queryset = models.CanonSiteConf.filter_manager.filter_qs().filter(superseded=False) serializer_class = serializers.CanonSiteConfReadSerializer filterset_class = filters.CanonSiteConfFilter - permission_class = [permissions.IsAuthenticated] - http_method_names = ('get',) + filter_permissions = "ref_site_observation__experiment__experiment_upload__project" -class XtalformSites(viewsets.ModelViewSet): +class XtalformSiteView(ISPyBSafeQuerySet): queryset = models.XtalformSite.filter_manager.filter_qs().filter(superseded=False) serializer_class = serializers.XtalformSiteReadSerializer filterset_class = filters.XtalformSiteFilter - permission_class = [permissions.IsAuthenticated] - http_method_names = ('get',) + filter_permissions = "canon_site__ref_conf_site__ref_site_observation__experiment__experiment_upload__project" class JobFileTransferView(viewsets.ModelViewSet): @@ -1733,7 +1776,7 @@ def get_serializer_class(self): def create(self, request): """Method to handle POST request""" - logger.info('+ JobFileTransfer.post') + logger.info('+ JobFileTransferView.post') # Only authenticated users can transfer files to sqonk user = self.request.user if not user.is_authenticated: @@ -1914,7 +1957,7 @@ def get_serializer_class(self): return serializers.JobOverrideWriteSerializer def create(self, request): - logger.info('+ JobOverride.post') + logger.info('+ JobOverrideView.post') # Only authenticated users can transfer files to sqonk user = self.request.user if not user.is_authenticated: @@ -1922,6 +1965,9 @@ def create(self, request): 'error': 'Only authenticated users can provide Job overrides' } return Response(content, status=status.HTTP_403_FORBIDDEN) + if not user.is_staff: + content = {'error': 'Only STAFF (Admin) users can provide Job overrides'} + return Response(content, status=status.HTTP_403_FORBIDDEN) # Override is expected to be a JSON string, # but protect against format issues @@ -1955,7 +2001,7 @@ def create(self, request): class JobRequestView(APIView): def get(self, request): - logger.info('+ JobRequest.get') + logger.info('+ JobRequestView.get') user = self.request.user if not user.is_authenticated: @@ -1978,16 +2024,22 @@ def get(self, request): snapshot_id = request.query_params.get('snapshot', None) if snapshot_id: - logger.info('+ JobRequest.get snapshot_id=%s', snapshot_id) + logger.info('+ JobRequestView.get snapshot_id=%s', snapshot_id) job_requests = models.JobRequest.objects.filter(snapshot=int(snapshot_id)) else: - logger.info('+ JobRequest.get snapshot_id=(unset)') + logger.info('+ JobRequestView.get snapshot_id=(unset)') job_requests = models.JobRequest.objects.all() for jr in job_requests: + # Skip any JobRequests the user does not have access to + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + user, [jr.project.title] + ): + continue + # An opportunity to update JobRequest timestamps? if not jr.job_has_finished(): logger.info( - '+ JobRequest.get (id=%s) has not finished (job_status=%s)', + '+ JobRequestView.get (id=%s) has not finished (job_status=%s)', jr.id, jr.job_status, ) @@ -1996,7 +2048,7 @@ def get(self, request): # To get the current status. To do this we'll need # the 'callback context' we supplied when launching the Job. logger.info( - '+ JobRequest.get (id=%s, code=%s) getting update from Squonk...', + '+ JobRequestView.get (id=%s, code=%s) getting update from Squonk...', jr.id, jr.code, ) @@ -2006,14 +2058,14 @@ def get(self, request): # 'LOST', 'SUCCESS' or 'FAILURE' if not sq2a_rv.success: logger.warning( - '+ JobRequest.get (id=%s, code=%s) check failed (%s)', + '+ JobRequestView.get (id=%s, code=%s) check failed (%s)', jr.id, jr.code, sq2a_rv.msg, ) elif sq2a_rv.success and sq2a_rv.msg: logger.info( - '+ JobRequest.get (id=%s, code=%s) new status is (%s)', + '+ JobRequestView.get (id=%s, code=%s) new status is (%s)', jr.id, jr.code, sq2a_rv.msg, @@ -2028,7 +2080,7 @@ def get(self, request): jr.save() else: logger.info( - '+ JobRequest.get (id=%s, code=%s) is (probably) still running', + '+ JobRequestView.get (id=%s, code=%s) is (probably) still running', jr.id, jr.code, ) @@ -2037,7 +2089,7 @@ def get(self, request): results.append(serializer.data) num_results = len(results) - logger.info('+ JobRequest.get num_results=%s', num_results) + logger.info('+ JobRequestView.get num_results=%s', num_results) # Simulate the original paged API response... content = { @@ -2049,7 +2101,7 @@ def get(self, request): return Response(content, status=status.HTTP_200_OK) def post(self, request): - logger.info('+ JobRequest.post') + logger.info('+ JobRequestView.post') # Only authenticated users can create squonk job requests # (unless 'AUTHENTICATE_UPLOAD' is False in settings.py) user = self.request.user @@ -2084,14 +2136,14 @@ def post(self, request): return Response(content, status=status.HTTP_404_NOT_FOUND) # The user must be a member of the target access string. # (when AUTHENTICATE_UPLOAD is set) - if settings.AUTHENTICATE_UPLOAD: - ispyb_safe_query_set = ISpyBSafeQuerySet() - user_proposals = ispyb_safe_query_set.get_proposals_for_user( - user, restrict_to_membership=True + if ( + settings.AUTHENTICATE_UPLOAD + and not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + user, [project.title] ) - if project.title not in user_proposals: - content = {'error': f"You are not a member of '{project.title}'"} - return Response(content, status=status.HTTP_403_FORBIDDEN) + ): + content = {'error': f"You are not a member of '{project.title}'"} + return Response(content, status=status.HTTP_403_FORBIDDEN) # Check the user can use this Squonk2 facility. # To do this we need to setup a couple of API parameter objects. @@ -2456,7 +2508,7 @@ def get(self, request): return Response(ok_response) -class ServiceState(View): +class ServiceStateView(View): def get(self, *args, **kwargs): """Poll external service status. @@ -2471,14 +2523,7 @@ def get(self, *args, **kwargs): """ # Unused arguments del args, kwargs - logger.debug("+ ServiceServiceState.State.get called") - service_string = settings.ENABLE_SERVICE_STATUS - logger.debug("Service string: %s", service_string) - - services = [k for k in service_string.split(":") if k != ""] - logger.debug("Services ordered: %s", services) - - service_states = get_service_state(services) - - return JsonResponse({"service_states": service_states}) + return JsonResponse( + {"service_states": list(Service.data_manager.to_frontend())} + ) diff --git a/xcdb/schema.py b/xcdb/schema.py deleted file mode 100644 index 3ae0bf02..00000000 --- a/xcdb/schema.py +++ /dev/null @@ -1,117 +0,0 @@ -import graphene -from graphene_django.rest_framework.mutation import SerializerMutation -from xchem_db.serializers import ( - CompoundsSerializer, - CrystalSerializer, - DataProcessingSerializer, - DimpleSerializer, - FragspectCrystalSerializer, - LabSerializer, - PanddaAnalysisSerializer, - PanddaEventSerializer, - PanddaRunSerializer, - PanddaSiteSerializer, - ProasisOutSerializer, - ReferenceSerializer, - RefinementSerializer, - SoakdbFilesSerializer, - TargetSerializer, -) - -relay = graphene.relay - - -class Target(SerializerMutation): - serializer_class = TargetSerializer - interfaces = (relay.Node,) - - -class Compounds(SerializerMutation): - serializer_class = CompoundsSerializer - interfaces = (relay.Node,) - - -class Reference(SerializerMutation): - serializer_class = ReferenceSerializer - interfaces = (relay.Node,) - - -class SoakdbFiles(SerializerMutation): - serializer_class = SoakdbFilesSerializer - interfaces = (relay.Node,) - - -class Crystal(SerializerMutation): - serializer_class = CrystalSerializer - interfaces = (relay.Node,) - - -class DataProcessing(SerializerMutation): - serializer_class = DataProcessingSerializer - interfaces = (relay.Node,) - - -class Dimple(SerializerMutation): - serializer_class = DimpleSerializer - interfaces = (relay.Node,) - - -class Lab(SerializerMutation): - serializer_class = LabSerializer - interfaces = (relay.Node,) - - -class Refinement(SerializerMutation): - serializer_class = RefinementSerializer - interfaces = (relay.Node,) - - -class PanddaAnalysis(SerializerMutation): - serializer_class = PanddaAnalysisSerializer - interfaces = (relay.Node,) - - -class PanddaRun(SerializerMutation): - serializer_class = PanddaRunSerializer - interfaces = (relay.Node,) - - -class PanddaSite(SerializerMutation): - serializer_class = PanddaSiteSerializer - interfaces = (relay.Node,) - - -class PanddaEvent(SerializerMutation): - serializer_class = PanddaEventSerializer - interfaces = (relay.Node,) - - -class ProasisOut(SerializerMutation): - serializer_class = ProasisOutSerializer - interfaces = (relay.Node,) - - -class Fragspect(SerializerMutation): - serializer_class = FragspectCrystalSerializer - interfaces = (relay.Node,) - - -class Query(graphene.ObjectType): - target = graphene.list(Target) - compounds = graphene.list(Compounds) - reference = graphene.list(Reference) - soakdb_files = graphene.list(SoakdbFiles) - crystal = graphene.list(Crystal) - data_processing = graphene.list(DataProcessing) - dimple = graphene.list(Dimple) - lab = graphene.list(Lab) - refinement = graphene.list(Refinement) - pandda_analysis = graphene.list(PanddaAnalysis) - pandda_run = graphene.list(PanddaRun) - pandda_site = graphene.list(PanddaSite) - pandda_event = graphene.list(PanddaEvent) - proasis_out = graphene.list(ProasisOut) - fragspect = graphene.list(Fragspect) - - -schema = graphene.Schema(query=Query) diff --git a/xcdb/urls.py b/xcdb/urls.py deleted file mode 100644 index 878be92b..00000000 --- a/xcdb/urls.py +++ /dev/null @@ -1,33 +0,0 @@ -from django.conf.urls import include -from django.urls import path -from rest_framework.authtoken import views as drf_views -from rest_framework.routers import DefaultRouter - -from xcdb.views import ( - CrystalView, - DataProcessingView, - DimpleView, - FragspectCrystalView, - LabView, - PanddaEventStatsView, - PanddaEventView, - ProasisOutView, - RefinementView, -) - -router = DefaultRouter() - -router.register(r'crystal', CrystalView) -router.register(r'dataproc', DataProcessingView) -router.register(r'dimple', DimpleView) -router.register(r'lab', LabView) -router.register(r'refinement', RefinementView) -router.register(r'pandda_event', PanddaEventView) -router.register(r'pandda_event_stats', PanddaEventStatsView) -router.register(r'proasis_out', ProasisOutView) -router.register(r'fragspect', FragspectCrystalView) - -urlpatterns = [ - path("", include(router.urls)), - path("auth", drf_views.obtain_auth_token, name="auth"), -] diff --git a/xcdb/views.py b/xcdb/views.py deleted file mode 100644 index 9982c092..00000000 --- a/xcdb/views.py +++ /dev/null @@ -1,182 +0,0 @@ -from xchem_db.models import ( - Crystal, - DataProcessing, - Dimple, - Lab, - PanddaEvent, - PanddaEventStats, - ProasisOut, - Refinement, -) -from xchem_db.serializers import ( - CrystalSerializer, - DataProcessingSerializer, - DimpleSerializer, - FragspectCrystalSerializer, - LabSerializer, - PanddaEventSerializer, - PanddaEventStatsSerializer, - ProasisOutSerializer, - RefinementSerializer, -) - -from api.security import ISpyBSafeQuerySet - - -class CrystalView(ISpyBSafeQuerySet): - queryset = Crystal.objects.filter() - filter_permissions = "visit__proposal" - serializer_class = CrystalSerializer - filter_fields = ( - "crystal_name", - "target__target_name", - "compound__smiles", - "visit__filename", - "visit__proposal__proposal", - "visit__visit", - ) - - -class DataProcessingView(ISpyBSafeQuerySet): - queryset = DataProcessing.objects.filter() - filter_permissions = "crystal_name__visit__proposal" - serializer_class = DataProcessingSerializer - filter_fields = ( - "crystal_name__crystal_name", - "crystal_name__target__target_name", - "crystal_name__compound__smiles", - "crystal_name__visit__filename", - "crystal_name__visit__proposal__proposal", - "crystal_name__visit__visit", - ) - - -class DimpleView(ISpyBSafeQuerySet): - queryset = Dimple.objects.filter() - filter_permissions = "crystal_name__visit__proposal" - serializer_class = DimpleSerializer - filter_fields = ( - "crystal_name__crystal_name", - "crystal_name__target__target_name", - "crystal_name__compound__smiles", - "crystal_name__visit__filename", - "crystal_name__visit__proposal__proposal", - "crystal_name__visit__visit", - "reference__reference_pdb", - ) - - -class LabView(ISpyBSafeQuerySet): - queryset = Lab.objects.filter() - filter_permissions = "crystal_name__visit__proposal" - serializer_class = LabSerializer - filter_fields = ( - "crystal_name__crystal_name", - "crystal_name__target__target_name", - "crystal_name__compound__smiles", - "crystal_name__visit__filename", - "crystal_name__visit__proposal__proposal", - "crystal_name__visit__visit", - "data_collection_visit", - "library_name", - "library_plate", - ) - - -class RefinementView(ISpyBSafeQuerySet): - queryset = Refinement.objects.filter() - filter_permissions = "crystal_name__visit__proposal" - serializer_class = RefinementSerializer - filter_fields = ( - "crystal_name__crystal_name", - "crystal_name__target__target_name", - "crystal_name__compound__smiles", - "crystal_name__visit__filename", - "crystal_name__visit__proposal__proposal", - "crystal_name__visit__visit", - "outcome", - ) - - -class PanddaEventView(ISpyBSafeQuerySet): - queryset = PanddaEvent.objects.filter() - filter_permissions = "crystal__visit__proposal" - serializer_class = PanddaEventSerializer - filter_fields = ( - "crystal__crystal_name", - "crystal__target__target_name", - "crystal__compound__smiles", - "crystal__visit__filename", - "crystal__visit__proposal__proposal", - "crystal__visit__visit", - # "pandda_run__pandda_analysis__pandda_dir", - # "pandda_run__pandda_log", - # "pandda_run__sites_file", - # "pandda_run__events_file", - # "pandda_run__input_dir", - # "site__site", - # "event", - # "lig_id", - # "pandda_event_map_native", - # "pandda_model_pdb", - # "pandda_input_mtz", - # "pandda_input_pdb", - ) - - -class PanddaEventStatsView(ISpyBSafeQuerySet): - queryset = PanddaEventStats.objects.filter() - filter_permissions = 'event__crystal__visit__proposal' - serializer_class = PanddaEventStatsSerializer - filter_fields = ( - "event__crystal__crystal_name", - "event__crystal__target__target_name", - "event__crystal__compound__smiles", - "event__crystal__visit__filename", - "event__crystal__visit__proposal__proposal", - "event__crystal__visit__visit", - ) - - -class ProasisOutView(ISpyBSafeQuerySet): - queryset = ProasisOut.objects.filter() - filter_permissions = "crystal__visit__proposal" - serializer_class = ProasisOutSerializer - filter_fields = ( - "crystal__crystal_name", - "crystal__target__target_name", - "crystal__compound__smiles", - "crystal__visit__filename", - "crystal__visit__proposal__proposal", - "crystal__visit__visit", - "proasis__strucid", - "proasis__crystal_name__crystal_name", - "proasis__crystal_name__target__target_name", - "proasis__crystal_name__compound__smiles", - "proasis__crystal_name__visit__filename", - "proasis__crystal_name__visit__proposal__proposal", - "proasis__crystal_name__visit__visit", - "proasis__refinement__crystal_name__crystal_name", - "proasis__refinement__crystal_name__target__target_name", - "proasis__refinement__crystal_name__compound__smiles", - "proasis__refinement__crystal_name__visit__filename", - "proasis__refinement__crystal_name__visit__proposal__proposal", - "proasis__refinement__crystal_name__visit__visit", - "proasis__refinement__outcome", - "root", - "start", - ) - - -class FragspectCrystalView(ISpyBSafeQuerySet): - queryset = PanddaEvent.objects.filter().prefetch_related( - 'crystal__target', - 'crystal__compound', - 'crystal', - 'site', - 'refinement', - 'data_proc', - ) - serializer_class = FragspectCrystalSerializer - filter_fields = {'crystal__target__target_name': ['iexact']} - filter_permissions = "crystal__visit__proposal"