diff --git a/.github/workflows/build-dev.yaml b/.github/workflows/build-dev.yaml index 689f42f5..4bb3aca3 100644 --- a/.github/workflows/build-dev.yaml +++ b/.github/workflows/build-dev.yaml @@ -85,12 +85,11 @@ jobs: with: context: . tags: ${{ steps.vars.outputs.BE_NAMESPACE }}/fragalysis-backend:${{ env.GITHUB_REF_SLUG }} - - name: Test - run: > - docker-compose -f docker-compose.test.yml up - --build - --exit-code-from tests - --abort-on-container-exit + - name: Test (docker compose) + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: ./docker-compose.test.yml + up-flags: --build --exit-code-from tests --abort-on-container-exit env: BE_NAMESPACE: ${{ steps.vars.outputs.BE_NAMESPACE }} BE_IMAGE_TAG: ${{ env.GITHUB_REF_SLUG }} diff --git a/.github/workflows/build-production.yaml b/.github/workflows/build-production.yaml index b49d06df..11cec835 100644 --- a/.github/workflows/build-production.yaml +++ b/.github/workflows/build-production.yaml @@ -134,12 +134,11 @@ jobs: tags: | ${{ steps.vars.outputs.BE_NAMESPACE }}/fragalysis-backend:${{ steps.vars.outputs.tag }} ${{ steps.vars.outputs.BE_NAMESPACE }}/fragalysis-backend:stable - - name: Test - run: > - docker-compose -f docker-compose.test.yml up - --build - --exit-code-from tests - --abort-on-container-exit + - name: Test (docker compose) + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: ./docker-compose.test.yml + up-flags: --build --exit-code-from tests --abort-on-container-exit env: BE_NAMESPACE: ${{ steps.vars.outputs.BE_NAMESPACE }} BE_IMAGE_TAG: ${{ steps.vars.outputs.tag }} diff --git a/.github/workflows/build-staging.yaml b/.github/workflows/build-staging.yaml index 35b70ff0..d76fe21a 100644 --- a/.github/workflows/build-staging.yaml +++ b/.github/workflows/build-staging.yaml @@ -154,12 +154,11 @@ jobs: with: context: . tags: ${{ steps.vars.outputs.BE_NAMESPACE }}/fragalysis-backend:${{ steps.vars.outputs.tag }} - - name: Test - run: > - docker-compose -f docker-compose.test.yml up - --build - --exit-code-from tests - --abort-on-container-exit + - name: Test (docker compose) + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: ./docker-compose.test.yml + up-flags: --build --exit-code-from tests --abort-on-container-exit env: BE_NAMESPACE: ${{ steps.vars.outputs.BE_NAMESPACE }} BE_IMAGE_TAG: ${{ steps.vars.outputs.tag }} diff --git a/api/security.py b/api/security.py index 9c95d86f..26d8a942 100644 --- a/api/security.py +++ b/api/security.py @@ -185,7 +185,7 @@ def ping_configured_connector() -> bool: return conn is not None -class ISpyBSafeQuerySet(viewsets.ReadOnlyModelViewSet): +class ISPyBSafeQuerySet(viewsets.ReadOnlyModelViewSet): """ This ISpyBSafeQuerySet, which inherits from the DRF viewsets.ReadOnlyModelViewSet, is used for all views that need to yield (filter) view objects based on a @@ -216,7 +216,7 @@ def get_queryset(self): q_filter = self._get_q_filter(proposal_list) return self.queryset.filter(q_filter).distinct() - def _get_open_proposals(self): + def get_open_proposals(self): """ Returns the set of proposals anybody can access. These consist of any Projects that are marked "open_to_public" @@ -226,6 +226,7 @@ def _get_open_proposals(self): Project.objects.filter(open_to_public=True).values_list("title", flat=True) ) open_proposals.update(settings.PUBLIC_TAS_LIST) + # End Temporary Test Code (1247) return open_proposals def _get_proposals_for_user_from_django(self, user): @@ -341,36 +342,67 @@ def _get_proposals_from_connector(self, user, conn): ) CachedContent.set_content(user.username, prop_id_set) - def user_is_member_of_any_given_proposals(self, user, proposals): + def user_is_member_of_target( + self, user, target, restrict_public_to_membership=True + ): + """ + Returns true if the user has access to any proposal the target belongs to. + """ + target_proposals = [p.title for p in target.project_id.all()] + user_proposals = self.get_proposals_for_user( + user, restrict_public_to_membership=restrict_public_to_membership + ) + is_member = any(proposal in user_proposals for proposal in target_proposals) + if not is_member: + logger.warning( + "Failed membership check user='%s' target='%s' target_proposals=%s", + user.username, + target.title, + target_proposals, + ) + return is_member + + def user_is_member_of_any_given_proposals( + self, user, proposals, restrict_public_to_membership=True + ): """ Returns true if the user has access to any proposal in the given - proposals list.Only one needs to match for permission to be granted. - We 'restrict_to_membership' to only consider proposals the user + proposals list. Only one needs to match for permission to be granted. + We 'restrict_public_to_membership' to only consider proposals the user has explicit membership. """ - user_proposals = self.get_proposals_for_user(user, restrict_to_membership=True) - return any(proposal in user_proposals for proposal in proposals) + user_proposals = self.get_proposals_for_user( + user, restrict_public_to_membership=restrict_public_to_membership + ) + is_member = any(proposal in user_proposals for proposal in proposals) + if not is_member: + logger.warning( + "Failed membership check user='%s' proposals=%s", + user.username, + proposals, + ) + return is_member - def get_proposals_for_user(self, user, restrict_to_membership=False): + def get_proposals_for_user(self, user, restrict_public_to_membership=False): """ Returns a list of proposals that the user has access to. - If 'restrict_to_membership' is set only those proposals/visits where the user + If 'restrict_public_to_membership' is set only those proposals/visits where the user is a member of the visit will be returned. Otherwise the 'public' - proposals/visits will also be returned. Typically 'restrict_to_membership' is + proposals/visits will also be returned. Typically 'restrict_public_to_membership' is used for uploads/changes - this allows us to implement logic that (say) only permits explicit members of public proposals to add/load data for that - project (restrict_to_membership=True), but everyone can 'see' public data - (restrict_to_membership=False). + project (restrict_public_to_membership=True), but everyone can 'see' public data + (restrict_public_to_membership=False). """ assert user proposals = set() ispyb_user = settings.ISPYB_USER logger.debug( - "ispyb_user=%s restrict_to_membership=%s (DISABLE_RESTRICT_PROPOSALS_TO_MEMBERSHIP=%s)", + "ispyb_user=%s restrict_public_to_membership=%s (DISABLE_RESTRICT_PROPOSALS_TO_MEMBERSHIP=%s)", ispyb_user, - restrict_to_membership, + restrict_public_to_membership, settings.DISABLE_RESTRICT_PROPOSALS_TO_MEMBERSHIP, ) if ispyb_user: @@ -384,10 +416,10 @@ def get_proposals_for_user(self, user, restrict_to_membership=False): # We have all the proposals where the user has authority. # Add open/public proposals? if ( - not restrict_to_membership + not restrict_public_to_membership or settings.DISABLE_RESTRICT_PROPOSALS_TO_MEMBERSHIP ): - proposals.update(self._get_open_proposals()) + proposals.update(self.get_open_proposals()) # Return the set() as a list() return list(proposals) @@ -411,9 +443,9 @@ def _get_q_filter(self, proposal_list): return Q(title__in=proposal_list) | Q(open_to_public=True) -class ISpyBSafeStaticFiles: +class ISPyBSafeStaticFiles: def get_queryset(self): - query = ISpyBSafeQuerySet() + query = ISPyBSafeQuerySet() query.request = self.request query.filter_permissions = self.permission_string query.queryset = self.model.objects.filter() @@ -459,7 +491,7 @@ def get_response(self): raise Http404 from exc -class ISpyBSafeStaticFiles2(ISpyBSafeStaticFiles): +class ISPyBSafeStaticFiles2(ISPyBSafeStaticFiles): def get_response(self): logger.info("+ get_response called with: %s", self.input_string) # it wasn't working because found two objects with test file name diff --git a/api/urls.py b/api/urls.py index 8f4d38ea..93536f9e 100644 --- a/api/urls.py +++ b/api/urls.py @@ -3,20 +3,18 @@ from rest_framework.authtoken import views as drf_views from rest_framework.routers import DefaultRouter -# from xcdb import views as xchem_views from hotspots import views as hostpot_views from hypothesis import views as hypo_views from scoring import views as score_views from viewer import views as viewer_views -from xcdb import views as xcdb_views router = DefaultRouter() # Register the basic data router.register("compounds", viewer_views.CompoundView, "compounds") router.register("targets", viewer_views.TargetView, "targets") router.register("projects", viewer_views.ProjectView) -router.register("session-projects", viewer_views.SessionProjectsView) -router.register("snapshots", viewer_views.SnapshotsView) +router.register("session-projects", viewer_views.SessionProjectView) +router.register("snapshots", viewer_views.SnapshotView) router.register("action-type", viewer_views.ActionTypeView) router.register("session-actions", viewer_views.SessionActionsView) router.register("snapshot-actions", viewer_views.SnapshotActionsView) @@ -26,7 +24,7 @@ # Compounds sets router.register("compound-sets", viewer_views.ComputedSetView) router.register("compound-molecules", viewer_views.ComputedMoleculesView) -router.register("numerical-scores", viewer_views.NumericalScoresView) +router.register("numerical-scores", viewer_views.NumericalScoreValuesView) router.register("text-scores", viewer_views.TextScoresView) router.register("compound-scores", viewer_views.CompoundScoresView, "compound-scores") router.register( @@ -65,16 +63,13 @@ # Get the information router.register("siteobservationannotation", score_views.SiteObservationAnnotationView) -# fragspect -router.register("fragspect", xcdb_views.FragspectCrystalView) - # discourse posts router.register( "discourse_post", viewer_views.DiscoursePostView, basename='discourse_post' ) # Take a dictionary and return a csv -router.register("dicttocsv", viewer_views.DictToCsv, basename='dicttocsv') +router.register("dicttocsv", viewer_views.DictToCSVView, basename='dicttocsv') # tags router.register("tag_category", viewer_views.TagCategoryView, basename='tag_category') @@ -92,36 +87,38 @@ # Download a zip file of the requested contents router.register( "download_structures", - viewer_views.DownloadStructures, + viewer_views.DownloadStructuresView, basename='download_structures', ) # Experiments and Experiment (XChemAlign) upload support router.register( "upload_target_experiments", - viewer_views.UploadTargetExperiments, + viewer_views.UploadExperimentUploadView, basename='upload_target_experiments', ) router.register( "download_target_experiments", - viewer_views.DownloadTargetExperiments, + viewer_views.DownloadExperimentUploadView, basename='download_target_experiments', ) router.register( "target_experiment_uploads", - viewer_views.TargetExperimentUploads, + viewer_views.ExperimentUploadView, basename='target_experiment_uploads', ) router.register( - "site_observations", viewer_views.SiteObservations, basename='site_observations' + "site_observations", viewer_views.SiteObservationView, basename='site_observations' +) +router.register("canon_sites", viewer_views.CanonSiteView, basename='canon_sites') +router.register( + "canon_site_confs", viewer_views.CanonSiteConfView, basename='canon_site_confs' ) -router.register("canon_sites", viewer_views.CanonSites, basename='canon_sites') router.register( - "canon_site_confs", viewer_views.CanonSiteConfs, basename='canon_site_confs' + "xtalform_sites", viewer_views.XtalformSiteView, basename='xtalform_sites' ) -router.register("xtalform_sites", viewer_views.XtalformSites, basename='xtalform_sites') router.register("poses", viewer_views.PoseView, basename='poses') # Squonk Jobs diff --git a/api/utils.py b/api/utils.py index f1196349..dc1a40e0 100644 --- a/api/utils.py +++ b/api/utils.py @@ -305,20 +305,16 @@ def parse_xenons(input_smi): return bond_ids, bond_colours, e_mol.GetMol() -def get_params(smiles, request): +def get_img_from_smiles(smiles, request): # try: smiles = canon_input(smiles) # except: # smiles = "" - height = None mol = None bond_id_list = [] highlightBondColors = {} - if "height" in request.GET: - height = int(request.GET["height"]) - width = None - if "width" in request.GET: - width = int(request.GET["width"]) + height = int(request.GET.get("height", "128")) + width = int(request.GET.get("width", "128")) if "atom_indices" in request.GET: mol = Chem.MolFromSmiles(smiles) bond_id_list, highlightBondColors, mol = parse_atom_ids( @@ -360,7 +356,7 @@ def get_highlighted_diffs(request): def mol_view(request): if "smiles" in request.GET: smiles = request.GET["smiles"].rstrip(".svg") - return get_params(smiles, request) + return get_img_from_smiles(smiles, request) else: return HttpResponse("Please insert SMILES") diff --git a/docs/source/API/api_intro.rst b/docs/source/API/api_intro.rst index 152a10cc..aed8f60f 100644 --- a/docs/source/API/api_intro.rst +++ b/docs/source/API/api_intro.rst @@ -23,7 +23,7 @@ The most common operations are: A more complex overview of these methods (and others) is available here: https://www.w3schools.com/tags/ref_httpmethods.asp -A more thorough explaination of RESTful APIs is available here: https://searchapparchitecture.techtarget.com/definition/RESTful-API +A more thorough explanation of RESTful APIs is available here: https://searchapparchitecture.techtarget.com/definition/RESTful-API Django REST framework (DRF) --------------------------- @@ -50,6 +50,67 @@ user to filter by (:code:`filter_fields`) and what :code:`Response` we want to p Finally, we need to specify an :code:`endpoint` (i.e. URL) that the :code:`View` is served at, so the user can make requests against the web service. +Basic style guidance +-------------------- +All new methods (functions) in **views** _should_ inherit from classes defined in DRF. +Some methods (functions) simply accept the :code:`request`, but this is older code. + +**Security** + +When writing a new API endpoint, it is important to consider the security implications +of the data you are making available to the user. Much of the data in fragalysis is +_open_, i.e. available to all. + +.. note:: + Open data is data associated with any :code:`Project` + that has the :code:`open_to_public` flag set to :code:`True`. + +As a general policy open data is visible to anyone, even if they are not authenticated +(logged in) but there is a policy that only logged-in users can modify or create open +data. More specifically users are required to be a member of (associated with) the +:code:`Project`` the object belongs to. To this end almost all endpoints +are required to check the object's Project (the Proposal/Visit) in order to determine +whether the User is permitted access. + +.. note:: + A user is 'associated' with a :code:`Project` (aka Proposal/Visit) if the security module in the + project's :code:`api` package is able to find the user associated with the + :code:`Project` by querying an external database. In diamond + this is an **ISPyB** MySQL database external to the stack installation whose + credentials are supplied using environment variables. + +API methods that provide access to data they must ensure that the user is authenticated +and must _strive_ to ensure that the user is associated with the :code:`Project` that +the data belongs to. + +In order to check whether the user has access to the object that is being created +or altered, each method must either identify the :code:`Project` that the object belongs to, +or there has to be a navigable path from any table record that might contain "sensitive" material +to one or more records in the :code:`Project` table. +Given a :code:`Project` is is a relatively simple task to check that the user +has been given access to us using the security module as described above, and +the code in the :code:`security` module relies on this pattern. + +These actions ar simplified through the use of the :code:`ISpyBSafeQuerySet` class +to filter objects when reading and the :code:`IsObjectProposalMember` class to +check the user has access to the object when creating or altering it. These classes +rely on the definition of :code:`filter_permissions` property to direct the +search to the object's :code:`Project`. + +View classes must generally inherit from :code:`ISpyBSafeQuerySet`, +which provides automatic filtering of objects. The :code:`ISpyBSafeQuerySet` +inherits from th :code:`ReadOnlyModelViewSet` view set. If a view also needs to provide +create, update or delete actions they should also inherit an appropriate +DRF **mixin**, adding support for a method so support the functionality that is +required: - + +- :code:`mixins.CreateModelMixin` - when supporting objects (POST) +- :code:`mixins.UpdateModelMixin` - when supporting objects (PATCH) +- :code:`mixins.DestroyModelMixin` - when supporting delete (DELETE) + +For further information refer to the `mixins`_ documentation on the DRF site. + + EXAMPLE - Model, Serializer, View and URL for Target model ---------------------------------------------------------- @@ -62,34 +123,11 @@ The Target model contains information about a protein target. In django, we defi from django.db import models class Target(models.Model): - """Django model to define a Target - a protein. - - Parameters - ---------- - title: CharField - The name of the target - init_date: DateTimeField - The date the target was initiated (autofield) - project_id: ManyToManyField - Links targets to projects for authentication - uniprot_id: Charfield - Optional field where a uniprot id can be stored - metadata: FileField - Optional file upload defining metadata about the target - can be used to add custom site labels - zip_archive: FileField - Link to zip file created from targets uploaded with the loader - """ - # The title of the project_id -> userdefined title = models.CharField(unique=True, max_length=200) - # The date it was made init_date = models.DateTimeField(auto_now_add=True) - # A field to link projects and targets together project_id = models.ManyToManyField(Project) - # Indicates the uniprot_id id for the target. Is a unique key uniprot_id = models.CharField(max_length=100, null=True) - # metadatafile containing sites info for download metadata = models.FileField(upload_to="metadata/", null=True, max_length=255) - # zip archive to download uploaded data from zip_archive = models.FileField(upload_to="archive/", null=True, max_length=255) @@ -129,6 +167,31 @@ can add extra fields, and add a method to define how we get the value of the fie :code:`Serializer` we have added the :code:`template_protein` field, and defined how we get its value with :code:`get_template_protein`. +**Models** + +Model definitions should avoid inline documentation, and instead use the django +:code:`help_text` parameter to provide this information. For example, +instead of doing this: - + +.. code-block:: python + + class Target(models.Model): + # The uniprot ID id for the target. A unique key + uniprot_id = models.CharField(max_length=100, null=True) + + +Do this: - + +.. code-block:: python + + class Target(models.Model): + uniprot_id = models.CharField( + max_length=100, + null=True, + help_text="The uniprot ID id for the target. A unique key", + ) + + **View** This :code:`View` returns a list of information about a specific target, if you pass the :code:`title` parameter to the @@ -141,7 +204,8 @@ of our standard views. Additionally, in the actual code, you will notice that :code:`TargetView(viewsets.ReadOnlyModelViewSet)` is replaced by :code:`TargetView(ISpyBSafeQuerySet)`. :code:`ISpyBSafeQuerySet` is a version of :code:`viewsets.ReadOnlyModelViewSet` -that includes an authentication method specific for the deployment of fragalysis at https://fragalysis.diamond.ac.uk +that includes an authentication method that filters records based omn a user's +membership of the object's :code:`project`. .. code-block:: python @@ -150,61 +214,11 @@ that includes an authentication method specific for the deployment of fragalysis from viewer.models import Target class TargetView(viewsets.ReadOnlyModelViewSet): - """ DjagnoRF view to retrieve info about targets - - Methods - ------- - url: - api/targets - queryset: - `viewer.models.Target.objects.filter()` - filter fields: - - `viewer.models.Target.title` - ?title= - returns: JSON - - id: id of the target object - - title: name of the target - - project_id: list of the ids of the projects the target is linked to - - protein_set: list of the ids of the protein sets the target is linked to - - template_protein: the template protein displayed in fragalysis front-end for this target - - metadata: link to the metadata file for the target if it was uploaded - - zip_archive: link to the zip archive of the uploaded data - - example output: - - .. code-block:: javascript - - "results": [ - { - "id": 62, - "title": "Mpro", - "project_id": [ - 2 - ], - "protein_set": [ - 29281, - 29274, - 29259, - 29305, - ..., - ], - "template_protein": "/media/pdbs/Mpro-x10417_0_apo.pdb", - "metadata": "http://fragalysis.diamond.ac.uk/media/metadata/metadata_2FdP5OJ.csv", - "zip_archive": "http://fragalysis.diamond.ac.uk/media/targets/Mpro.zip" - } - ] - - """ queryset = Target.objects.filter() serializer_class = TargetSerializer filter_permissions = "project_id" filter_fields = ("title",) - -The docstring for this class is formatted in a way to allow a user or developer to easily read the docstring, and -understand the URL to query, how the information is queried by django, what fields can be queried against, and what -information is returned from a request against the views URL. All of the views in this documentation are written in the -same way. - **URL** Finally, we need to define where the view is served from, in context of the root (e.g. https://fragalysis.diamond.ac.uk) @@ -243,3 +257,5 @@ If we navigate to the URL :code:`/api/targets/?title=` we are This is a page automatically generated by DRF, and includes options to see what kinds of requests you can make against this endpoint. + +.. _mixins: https://www.django-rest-framework.org/tutorial/3-class-based-views/#using-mixins diff --git a/fragalysis/schema.py b/fragalysis/schema.py index ca8f4add..4f86bd4f 100644 --- a/fragalysis/schema.py +++ b/fragalysis/schema.py @@ -5,7 +5,6 @@ import hypothesis.schema import scoring.schema import viewer.schema -import xcdb.schema class Query( @@ -14,7 +13,6 @@ class Query( hotspots.schema.Query, pandda.schema.Query, scoring.schema.Query, - xcdb.schema.Query, graphene.ObjectType, ): # This class will inherit from multiple Queries diff --git a/fragalysis/settings.py b/fragalysis/settings.py index 9ce8991d..f5e20129 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -347,16 +347,6 @@ } } -if os.environ.get("BUILD_XCDB") == "yes": - DATABASES["xchem_db"] = { - "ENGINE": "django.db.backends.postgresql", - "NAME": os.environ.get("XCHEM_NAME", ""), - "USER": os.environ.get("XCHEM_USER", ""), - "PASSWORD": os.environ.get("XCHEM_PASSWORD", ""), - "HOST": os.environ.get("XCHEM_HOST", ""), - "PORT": os.environ.get("XCHEM_PORT", ""), - } - CHEMCENTRAL_DB_NAME = os.environ.get("CHEMCENT_DB_NAME", "UNKNOWN") if CHEMCENTRAL_DB_NAME != "UNKNOWN": DATABASES["chemcentral"] = { diff --git a/fragalysis/urls.py b/fragalysis/urls.py index 048f1252..2d44e47c 100644 --- a/fragalysis/urls.py +++ b/fragalysis/urls.py @@ -31,7 +31,6 @@ path("api/", include("api.urls")), path("media/", include("media_serve.urls")), path("scoring/", include("scoring.urls")), - path("xcdb/", include("xcdb.urls")), path("graphql/", GraphQLView.as_view(graphiql=True)), path('oidc/', include('mozilla_django_oidc.urls')), path( diff --git a/hotspots/views.py b/hotspots/views.py index a76bd4f4..b7e5b762 100644 --- a/hotspots/views.py +++ b/hotspots/views.py @@ -1,10 +1,10 @@ -from rest_framework import viewsets - +from api.security import ISPyBSafeQuerySet from hotspots.models import HotspotMap from hotspots.serializers import HotspotMapSerializer -class HotspotView(viewsets.ReadOnlyModelViewSet): +class HotspotView(ISPyBSafeQuerySet): queryset = HotspotMap.objects.all() serializer_class = HotspotMapSerializer filterset_fields = ("map_type", "target", "site_observation") + filter_permissions = "target__project_id" diff --git a/hypothesis/views.py b/hypothesis/views.py index ab1557d7..1872bfa6 100644 --- a/hypothesis/views.py +++ b/hypothesis/views.py @@ -1,5 +1,4 @@ -from rest_framework import viewsets - +from api.security import ISPyBSafeQuerySet from hypothesis.models import Interaction, InteractionPoint, TargetResidue from hypothesis.serializers import ( InteractionPointSerializer, @@ -8,7 +7,7 @@ ) -class InteractionView(viewsets.ReadOnlyModelViewSet): +class InteractionView(ISPyBSafeQuerySet): queryset = Interaction.objects.filter() serializer_class = InteractionSerializer filterset_fields = ( @@ -22,15 +21,18 @@ class InteractionView(viewsets.ReadOnlyModelViewSet): "prot_smarts", "mol_smarts", ) + filter_permissions = "interaction_point__targ_res__target_id__project_id" -class InteractionPointView(viewsets.ReadOnlyModelViewSet): +class InteractionPointView(ISPyBSafeQuerySet): queryset = InteractionPoint.objects.all() serializer_class = InteractionPointSerializer filterset_fields = ("site_observation", "protein_atom_name", "molecule_atom_name") + filter_permissions = "targ_res__target_id__project_id" -class TargetResidueView(viewsets.ReadOnlyModelViewSet): +class TargetResidueView(ISPyBSafeQuerySet): queryset = TargetResidue.objects.all() serializer_class = TargetResidueSerialzier filterset_fields = ("target_id", "res_name", "res_num", "chain_id") + filter_permissions = "target_id__project_id" diff --git a/media_serve/views.py b/media_serve/views.py index c44411fa..ce599aeb 100644 --- a/media_serve/views.py +++ b/media_serve/views.py @@ -1,6 +1,6 @@ import logging -from api.security import ISpyBSafeStaticFiles, ISpyBSafeStaticFiles2 +from api.security import ISPyBSafeStaticFiles, ISPyBSafeStaticFiles2 from viewer.models import SiteObservation, Target logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ def file_download(request, file_path): :return: the response (a redirect to nginx internal) """ logger.info("+ Received file_download file path: %s", file_path) - ispy_b_static = ISpyBSafeStaticFiles2() + ispy_b_static = ISPyBSafeStaticFiles2() # ispy_b_static = ISpyBSafeStaticFiles() ispy_b_static.model = SiteObservation ispy_b_static.request = request @@ -64,7 +64,7 @@ def tld_download(request, file_path): :return: the response (a redirect to nginx internal) """ logger.info("+ Received tld_download file path: %s", file_path) - ispy_b_static = ISpyBSafeStaticFiles2() + ispy_b_static = ISPyBSafeStaticFiles2() # ispy_b_static = ISpyBSafeStaticFiles() ispy_b_static.model = SiteObservation ispy_b_static.request = request @@ -88,7 +88,7 @@ def cspdb_download(request, file_path): :return: the response (a redirect to nginx internal) """ logger.info("+ Received cspdb_download file path: %s", file_path) - ispy_b_static = ISpyBSafeStaticFiles2() + ispy_b_static = ISPyBSafeStaticFiles2() ispy_b_static.model = SiteObservation ispy_b_static.request = request # the following 2 aren't used atm @@ -109,7 +109,7 @@ def bound_download(request, file_path): :param file_path: the file path we're getting from the static :return: the response (a redirect to nginx internal) """ - ispy_b_static = ISpyBSafeStaticFiles() + ispy_b_static = ISPyBSafeStaticFiles() ispy_b_static.model = SiteObservation ispy_b_static.request = request ispy_b_static.permission_string = "target_id__project_id" @@ -127,7 +127,7 @@ def map_download(request, file_path): :param file_path: the file path we're getting from the static :return: the response (a redirect to nginx internal) """ - ispy_b_static = ISpyBSafeStaticFiles() + ispy_b_static = ISPyBSafeStaticFiles() ispy_b_static.model = SiteObservation ispy_b_static.request = request ispy_b_static.permission_string = "target_id__project_id" @@ -163,7 +163,7 @@ def metadata_download(request, file_path): :param file_path: the file path we're getting from the static :return: the response (a redirect to nginx internal) """ - ispy_b_static = ISpyBSafeStaticFiles() + ispy_b_static = ISPyBSafeStaticFiles() ispy_b_static.model = Target ispy_b_static.request = request ispy_b_static.permission_string = "project_id" @@ -181,7 +181,7 @@ def archive_download(request, file_path): :param file_path: the file path we're getting from the static :return: the response (a redirect to nginx internal) """ - ispy_b_static = ISpyBSafeStaticFiles() + ispy_b_static = ISPyBSafeStaticFiles() ispy_b_static.model = Target ispy_b_static.request = request ispy_b_static.permission_string = "project_id" diff --git a/scoring/managers.py b/scoring/managers.py index cb2bd8b0..40cbdd5e 100644 --- a/scoring/managers.py +++ b/scoring/managers.py @@ -30,3 +30,106 @@ def filter_qs(self): def by_target(self, target): return self.get_queryset().filter_qs().filter(target=target.id) + + +class SiteObservationChoiceQueryset(QuerySet): + def filter_qs(self): + SiteObservationChoice = apps.get_model("scoring", "SiteObservationChoice") + qs = SiteObservationChoice.objects.prefetch_related( + "site_observation", + "site_observation__experiment", + "site_observation__experiment__experiment_upload", + "site_observation__experiment__experiment_upload__target", + ).annotate( + target=F("site_observation__experiment__experiment_upload__target"), + ) + + return qs + + +class SiteObservationChoiceDataManager(Manager): + def get_queryset(self): + return SiteObservationChoiceQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class CmpdChoiceQueryset(QuerySet): + def filter_qs(self): + CmpdChoice = apps.get_model("scoring", "CmpdChoice") + qs = CmpdChoice.objects.prefetch_related( + "cmpd_id__siteobservation", + "cmpd_id__siteobservation__experiment", + "cmpd_id__siteobservation__experiment__experiment_upload", + "cmpd_id__siteobservation__experiment__experiment_upload__target", + ).annotate( + target=F("cmpd_id__siteobservation__experiment__experiment_upload__target"), + ) + + return qs + + +class CmpdChoiceDataManager(Manager): + def get_queryset(self): + return CmpdChoiceQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class ViewSceneQueryset(QuerySet): + def filter_qs(self): + ViewScene = apps.get_model("scoring", "ViewScene") + qs = ViewScene.objects.prefetch_related( + "snapshot__session_project__target", + ).annotate( + target=F("snapshot__session_project__target"), + ) + + return qs + + +class ViewSceneDataManager(Manager): + def get_queryset(self): + return ViewSceneQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class SiteObservationAnnotationQueryset(QuerySet): + def filter_qs(self): + SiteObservationAnnotation = apps.get_model( + "scoring", "SiteObservationAnnotation" + ) + qs = SiteObservationAnnotation.objects.prefetch_related( + "site_observation", + "site_observation__experiment", + "site_observation__experiment__experiment_upload", + "site_observation__experiment__experiment_upload__target", + ).annotate( + target=F("site_observation__experiment__experiment_upload__target"), + ) + + return qs + + +class SiteObservationAnnotationDataManager(Manager): + def get_queryset(self): + return SiteObservationChoiceQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) diff --git a/scoring/models.py b/scoring/models.py index 39483897..dbaf42fe 100644 --- a/scoring/models.py +++ b/scoring/models.py @@ -3,7 +3,13 @@ from viewer.models import Compound, SiteObservation, Snapshot, Target -from .managers import ScoreChoiceDataManager +from .managers import ( + CmpdChoiceDataManager, + ScoreChoiceDataManager, + SiteObservationAnnotationDataManager, + SiteObservationChoiceDataManager, + ViewSceneDataManager, +) class ViewScene(models.Model): @@ -29,6 +35,9 @@ class ViewScene(models.Model): # for redirecting to project's snapshot snapshot = models.ForeignKey(Snapshot, null=True, on_delete=models.CASCADE) + objects = models.Manager() + filter_manager = ViewSceneDataManager() + class SiteObservationChoice(models.Model): """ @@ -50,6 +59,9 @@ class SiteObservationChoice(models.Model): # Score - score = models.FloatField(null=True) + objects = models.Manager() + filter_manager = SiteObservationChoiceDataManager() + class Meta: constraints = [ models.UniqueConstraint( @@ -68,6 +80,9 @@ class SiteObservationAnnotation(models.Model): annotation_type = models.CharField(max_length=50) annotation_text = models.CharField(max_length=100) + objects = models.Manager() + filter_manager = SiteObservationAnnotationDataManager() + class Meta: constraints = [ models.UniqueConstraint( @@ -128,6 +143,9 @@ class CmpdChoice(models.Model): # E.g. score = models.FloatField(null=True) + objects = models.Manager() + filter_manager = CmpdChoiceDataManager() + class Meta: unique_together = ("user_id", "cmpd_id", "choice_type") diff --git a/scoring/serializers.py b/scoring/serializers.py index 0bd21b67..3400fb58 100644 --- a/scoring/serializers.py +++ b/scoring/serializers.py @@ -8,9 +8,10 @@ SiteObservationGroup, ViewScene, ) +from viewer.serializers import ValidateProjectMixin -class ViewSceneSerializer(serializers.ModelSerializer): +class ViewSceneSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = ViewScene fields = ( @@ -25,25 +26,46 @@ class Meta: ) -class SiteObservationChoiceSerializer(serializers.ModelSerializer): +class SiteObservationChoiceSerializer( + ValidateProjectMixin, serializers.ModelSerializer +): class Meta: model = SiteObservationChoice - fields = ("id", "user", "site_observation", "choice_type", "score") + fields = ( + "id", + "user", + "site_observation", + "choice_type", + "score", + ) -class SiteObservationAnnotationSerializer(serializers.ModelSerializer): +class SiteObservationAnnotationSerializer( + ValidateProjectMixin, serializers.ModelSerializer +): class Meta: model = SiteObservationAnnotation - fields = ("id", "site_observation", "annotation_type", "annotation_text") + fields = ( + "id", + "site_observation", + "annotation_type", + "annotation_text", + ) -class CmpdChoiceSerializer(serializers.ModelSerializer): +class CmpdChoiceSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = CmpdChoice - fields = ("id", "user_id", "cmpd_id", "choice_type", "score") + fields = ( + "id", + "user_id", + "cmpd_id", + "choice_type", + "score", + ) -class ScoreChoiceSerializer(serializers.ModelSerializer): +class ScoreChoiceSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = ScoreChoice fields = ( @@ -56,7 +78,7 @@ class Meta: ) -class SiteObservationGroupSerializer(serializers.ModelSerializer): +class SiteObservationGroupSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = SiteObservationGroup fields = ( diff --git a/scoring/views.py b/scoring/views.py index af1d556e..1706b0a8 100644 --- a/scoring/views.py +++ b/scoring/views.py @@ -2,8 +2,9 @@ from django.http import HttpResponse from frag.conf.functions import generate_confs_for_vector -from rest_framework import viewsets +from rest_framework import mixins +from api.security import ISPyBSafeQuerySet from scoring.models import ( CmpdChoice, ScoreChoice, @@ -20,20 +21,33 @@ SiteObservationGroupSerializer, ViewSceneSerializer, ) +from viewer.permissions import IsObjectProposalMember -class ViewSceneView(viewsets.ModelViewSet): - queryset = ViewScene.objects.all().order_by("-modified") +class ViewSceneView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): + queryset = ViewScene.filter_manager.filter_qs().order_by("-modified") # filter_backends = (filters.DjangoFilterBackend,) serializer_class = ViewSceneSerializer filterset_fields = ("user_id", "uuid") + filter_permissions = "snapshot__session_project__target__project_id" + permission_classes = [IsObjectProposalMember] def put(self, request, *args, **kwargs): return self.partial_update(request, *args, **kwargs) -class SiteObservationChoiceView(viewsets.ModelViewSet): - queryset = SiteObservationChoice.objects.all() +class SiteObservationChoiceView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): + queryset = SiteObservationChoice.filter_manager.filter_qs() serializer_class = SiteObservationChoiceSerializer filterset_fields = ( "user", @@ -41,21 +55,42 @@ class SiteObservationChoiceView(viewsets.ModelViewSet): "site_observation__experiment__experiment_upload__target", "choice_type", ) + filter_permissions = "site_observation__experiment__experiment_upload__project" + permission_classes = [IsObjectProposalMember] -class SiteObservationAnnotationView(viewsets.ModelViewSet): - queryset = SiteObservationAnnotation.objects.all() +class SiteObservationAnnotationView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): + queryset = SiteObservationAnnotation.filter_manager.filter_qs() serializer_class = SiteObservationAnnotationSerializer filterset_fields = ("site_observation", "annotation_type") + filter_permissions = "site_observation__experiment__experiment_upload__project" + permission_classes = [IsObjectProposalMember] -class CmpdChoiceView(viewsets.ModelViewSet): - queryset = CmpdChoice.objects.all() +class CmpdChoiceView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): + queryset = CmpdChoice.filter_manager.filter_qs() serializer_class = CmpdChoiceSerializer filterset_fields = ("user_id", "cmpd_id", "choice_type") + filter_permissions = "cmpd_id__project_id" + permission_classes = [IsObjectProposalMember] -class ScoreChoiceView(viewsets.ModelViewSet): +class ScoreChoiceView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): queryset = ScoreChoice.filter_manager.filter_qs() serializer_class = ScoreChoiceSerializer filterset_fields = ( @@ -65,12 +100,21 @@ class ScoreChoiceView(viewsets.ModelViewSet): "site_observation__experiment__experiment_upload__target", "choice_type", ) + filter_permissions = "site_observation__experiment__experiment_upload__project" + permission_classes = [IsObjectProposalMember] -class SiteObservationGroupView(viewsets.ModelViewSet): +class SiteObservationGroupView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): queryset = SiteObservationGroup.objects.all() serializer_class = SiteObservationGroupSerializer filterset_fields = ("group_type", "site_observation", "target", "description") + filter_permissions = "target__project_id" + permission_classes = [IsObjectProposalMember] def gen_conf_from_vect(request): diff --git a/viewer/cset_upload.py b/viewer/cset_upload.py index 5db2ef7a..6dcd3edb 100644 --- a/viewer/cset_upload.py +++ b/viewer/cset_upload.py @@ -289,7 +289,33 @@ def create_mol(self, inchi, target, name=None) -> Compound: inchi_key=inchi_key, current_identifier=name, ) + # This is a new compound. + # We must now set relationships to the Proposal that it applies to. + # We do this by copying the relationships from the Target. + num_target_proposals = len(target.project_id.all()) + assert num_target_proposals > 0 + if num_target_proposals > 1: + logger.warning( + 'Compound Target %s has more than one Proposal (%d)', + target.title, + num_target_proposals, + ) + for project in target.project_set.all(): + cpd.project_id.add(project) cpd.save() + # This is a new compound. + # We must now set relationships to the Proposal that it applies to. + # We do this by copying the relationships from the Target. + num_target_proposals = len(target.project_id.all()) + assert num_target_proposals > 0 + if num_target_proposals > 1: + logger.warning( + 'Compound Target %s has more than one Proposal (%d)', + target.title, + num_target_proposals, + ) + for project in target.project_id.all(): + cpd.project_id.add(project) except MultipleObjectsReturned as exc: # NB! when processing new uploads, Compound is always # fetched by inchi_key, so this shouldn't ever create @@ -339,7 +365,6 @@ def set_mol( smiles = Chem.MolToSmiles(mol) inchi = Chem.inchi.MolToInchi(mol) molecule_name = mol.GetProp('_Name') - version = mol.GetProp('version') compound: Compound = self.create_mol( inchi, compound_set.target, name=molecule_name @@ -449,8 +474,9 @@ def set_mol( existing_computed_molecules.append(k) if len(existing_computed_molecules) == 1: - logger.info( - 'Using existing ComputedMolecule %s', existing_computed_molecules[0] + logger.warning( + 'Using existing ComputedMolecule %s and overwriting its metadata', + existing_computed_molecules[0], ) computed_molecule = existing_computed_molecules[0] elif len(existing_computed_molecules) > 1: @@ -485,7 +511,6 @@ def set_mol( computed_molecule.pdb = lhs_so # TODO: this is wrong computed_molecule.pdb_info = pdb_info - computed_molecule.version = version # Extract possible reference URL and Rationale # URLs have to be valid URLs and rationals must contain more than one word ref_url: Optional[str] = ( diff --git a/viewer/download_structures.py b/viewer/download_structures.py index 0a25850c..d1a3d8ce 100644 --- a/viewer/download_structures.py +++ b/viewer/download_structures.py @@ -104,6 +104,25 @@ def __init__(self, category): # fmt: on +class UploadTagSubquery(Subquery): + """Annotate SiteObservation with tag of given category""" + + def __init__(self, category): + # fmt: off + query = SiteObservationTag.objects.filter( + pk=Subquery( + SiteObvsSiteObservationTag.objects.filter( + site_observation=OuterRef(OuterRef('pk')), + site_obvs_tag__category=TagCategory.objects.get( + category=category, + ), + ).values('site_obvs_tag')[:1] + ) + ).values('upload_name')[0:1] + super().__init__(query) + # fmt: on + + class CuratedTagSubquery(Exists): """Annotate SiteObservation with tag of given category""" @@ -140,6 +159,10 @@ class ArchiveFile: 'ligand_pdb': {}, 'ligand_mol': {}, 'ligand_smiles': {}, + # additional ccp4 files, issue 1448 + 'event_file_crystallographic': {}, + 'diff_file_crystallographic': {}, + 'sigmaa_file_crystallographic': {}, }, 'molecules': { 'sdf_files': {}, @@ -423,14 +446,34 @@ def _metadata_file_zip(ziparchive, target, site_observations): logger.info('+ Processing metadata') annotations = {} - values = ['code', 'longcode', 'cmpd__compound_code', 'smiles', 'downloaded'] - header = ['Code', 'Long code', 'Compound code', 'Smiles', 'Downloaded'] + values = [ + 'code', + 'longcode', + 'experiment__code', + 'cmpd__compound_code', + 'smiles', + 'canon_site_conf__canon_site__centroid_res', + 'downloaded', + ] + header = [ + 'Code', + 'Long code', + 'Experiment code', + 'Compound code', + 'Smiles', + 'Centroid res', + 'Downloaded', + ] for category in TagCategory.objects.filter(category__in=TAG_CATEGORIES): tag = f'tag_{category.category.lower()}' + upload_tag = f'upload_tag_{category.category.lower()}' values.append(tag) - header.append(category.category) + header.append(f'{category.category} alias') annotations[tag] = TagSubquery(category.category) + values.append(upload_tag) + header.append(f'{category.category} upload name') + annotations[upload_tag] = UploadTagSubquery(category.category) pattern = re.compile(r'\W+') # non-alphanumeric characters for tag in SiteObservationTag.objects.filter( @@ -812,6 +855,7 @@ def _create_structures_dict(site_obvs, protein_params, other_params): ) ) else: + # file not in upload archive_path = str(apath.joinpath(param)) afile = [ @@ -820,12 +864,38 @@ def _create_structures_dict(site_obvs, protein_params, other_params): archive_path=archive_path, ) ] + else: logger.warning('Unexpected param: %s', param) continue zip_contents['proteins'][param][so.code] = afile + # add additional ccp4 files (issue 1448) + ccps = ('sigmaa_file', 'diff_file', 'event_file') + if param in ccps: + # these only come from siteobservation object + model_attr = getattr(so, param) + if model_attr and model_attr != 'None': + apath = Path('aligned_files').joinpath(so.code) + ccp_path = Path(model_attr.name) + path = ccp_path.parent.joinpath( + f'{ccp_path.stem}_crystallographic{ccp_path.suffix}' + ) + archive_path = str( + apath.joinpath(path.parts[-1].replace(so.longcode, so.code)) + ) + + afile = [ + ArchiveFile( + path=str(path), + archive_path=archive_path, + ) + ] + zip_contents['proteins'][f'{param}_crystallographic'][ + so.code + ] = afile + zip_contents['molecules']['single_sdf_file'] = other_params['single_sdf_file'] zip_contents['molecules']['sdf_info'] = other_params['sdf_info'] diff --git a/viewer/managers.py b/viewer/managers.py index 12d9937d..29985fe3 100644 --- a/viewer/managers.py +++ b/viewer/managers.py @@ -315,3 +315,101 @@ def filter_qs(self): def by_target(self, target): return self.get_queryset().filter_qs().filter(target=target.id) + + +class SnapshotQueryset(QuerySet): + def filter_qs(self): + Snapshot = apps.get_model("viewer", "Snapshot") + qs = Snapshot.objects.prefetch_related( + "session_project", + "session_project__target", + ).annotate( + target=F("session_project__target"), + ) + + return qs + + +class SnapshotDataManager(Manager): + def get_queryset(self): + return SnapshotQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class SnapshotActionsQueryset(QuerySet): + def filter_qs(self): + SnapshotActions = apps.get_model("viewer", "SnapshotActions") + qs = SnapshotActions.objects.prefetch_related( + "session_project", + "session_project__target", + ).annotate( + target=F("session_project__target"), + ) + + return qs + + +class SnapshotActionsDataManager(Manager): + def get_queryset(self): + return SnapshotActionsQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class SessionActionsQueryset(QuerySet): + def filter_qs(self): + SessionActions = apps.get_model("viewer", "SessionActions") + qs = SessionActions.objects.prefetch_related( + "session_project", + "session_project__target", + ).annotate( + target=F("session_project__target"), + ) + + return qs + + +class SessionActionsDataManager(Manager): + def get_queryset(self): + return SessionActionsQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) + + +class CompoundIdentifierQueryset(QuerySet): + def filter_qs(self): + CompoundIdentifier = apps.get_model("viewer", "CompoundIdentifier") + qs = CompoundIdentifier.objects.prefetch_related( + "cmpd_id__siteobservation", + "cmpd_id__siteobservation__experiment", + "cmpd_id__siteobservation__experiment__experiment_upload", + "cmpd_id__siteobservation__experiment__experiment_upload__target", + ).annotate( + target=F("cmpd_id__siteobservation__experiment__experiment_upload__target"), + ) + + return qs + + +class CompoundIdentifierDataManager(Manager): + def get_queryset(self): + return CompoundIdentifierQueryset(self.model, using=self._db) + + def filter_qs(self): + return self.get_queryset().filter_qs() + + def by_target(self, target): + return self.get_queryset().filter_qs().filter(target=target.id) diff --git a/viewer/migrations/0059_remove_computedmolecule_version.py b/viewer/migrations/0059_remove_computedmolecule_version.py new file mode 100644 index 00000000..c6baece9 --- /dev/null +++ b/viewer/migrations/0059_remove_computedmolecule_version.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.25 on 2024-07-10 08:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0058_auto_20240614_1016'), + ] + + operations = [ + migrations.RemoveField( + model_name='computedmolecule', + name='version', + ), + ] diff --git a/viewer/migrations/0060_canonsite_centroid_res.py b/viewer/migrations/0060_canonsite_centroid_res.py new file mode 100644 index 00000000..97150507 --- /dev/null +++ b/viewer/migrations/0060_canonsite_centroid_res.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-07-29 12:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('viewer', '0059_remove_computedmolecule_version'), + ] + + operations = [ + migrations.AddField( + model_name='canonsite', + name='centroid_res', + field=models.TextField(null=True), + ), + ] diff --git a/viewer/models.py b/viewer/models.py index 85678ad0..95c58650 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -17,10 +17,14 @@ CanonSiteConfDataManager, CanonSiteDataManager, CompoundDataManager, + CompoundIdentifierDataManager, ExperimentDataManager, PoseDataManager, QuatAssemblyDataManager, + SessionActionsDataManager, SiteObservationDataManager, + SnapshotActionsDataManager, + SnapshotDataManager, XtalformDataManager, XtalformQuatAssemblyDataManager, XtalformSiteDataManager, @@ -196,6 +200,15 @@ def get_upload_path(self): .joinpath(self.upload_data_dir) ) + def get_download_path(self): + """The path to the original uploaded file, used during downloads""" + return ( + Path(settings.MEDIA_ROOT) + .joinpath(settings.TARGET_LOADER_MEDIA_DIRECTORY) + .joinpath(self.task_id) + .joinpath(Path(str(self.file))) + ) + class Experiment(models.Model): experiment_upload = models.ForeignKey(ExperimentUpload, on_delete=models.CASCADE) @@ -396,6 +409,7 @@ class CanonSite(Versionable, models.Model): canon_site_num = models.IntegerField( null=True, help_text="numeric canon site id (enumerated on creation)" ) + centroid_res = models.TextField(null=True) objects = models.Manager() filter_manager = CanonSiteDataManager() @@ -567,6 +581,9 @@ class CompoundIdentifier(models.Model): url = models.URLField(max_length=URL_LENGTH, null=True) name = models.CharField(max_length=NAME_LENGTH) + objects = models.Manager() + filter_manager = CompoundIdentifierDataManager() + def __str__(self) -> str: return f"{self.name}" @@ -682,6 +699,9 @@ class SessionActions(models.Model): last_update_date = models.DateTimeField(default=timezone.now) actions = models.JSONField(encoder=DjangoJSONEncoder) + objects = models.Manager() + filter_manager = SessionActionsDataManager() + def __str__(self) -> str: return f"{self.author}" @@ -737,6 +757,9 @@ class Snapshot(models.Model): help_text='Optional JSON field containing name/value pairs for future use', ) + objects = models.Manager() + filter_manager = SnapshotDataManager() + def __str__(self) -> str: return f"{self.title}" @@ -773,6 +796,9 @@ class SnapshotActions(models.Model): last_update_date = models.DateTimeField(default=timezone.now) actions = models.JSONField(encoder=DjangoJSONEncoder) + objects = models.Manager() + filter_manager = SnapshotActionsDataManager() + def __str__(self) -> str: return f"{self.author}" @@ -1027,7 +1053,6 @@ class ComputedMolecule(models.Model): max_length=255, help_text="Link to pdb file; user-uploaded pdb or pdb.experiment.pdb_info", ) - version = models.PositiveSmallIntegerField(null=False, default=1) def __str__(self) -> str: return f"{self.smiles}" diff --git a/viewer/permissions.py b/viewer/permissions.py index 95c975e6..b8ebab77 100644 --- a/viewer/permissions.py +++ b/viewer/permissions.py @@ -1,9 +1,9 @@ from rest_framework import permissions from rest_framework.exceptions import PermissionDenied -from api.security import ISpyBSafeQuerySet +from api.security import ISPyBSafeQuerySet -_ISPYB_SAFE_QUERY_SET = ISpyBSafeQuerySet() +_ISPYB_SAFE_QUERY_SET = ISPyBSafeQuerySet() class IsObjectProposalMember(permissions.BasePermission): diff --git a/viewer/serializers.py b/viewer/serializers.py index 7b15bf6b..b12c880c 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -12,8 +12,9 @@ from rdkit import Chem from rdkit.Chem import Descriptors from rest_framework import serializers +from rest_framework.exceptions import PermissionDenied -from api.security import ISpyBSafeQuerySet +from api.security import ISPyBSafeQuerySet from api.utils import draw_mol, validate_tas from viewer import models from viewer.target_loader import XTALFORMS_FILE @@ -22,7 +23,70 @@ logger = logging.getLogger(__name__) -_ISPYB_SAFE_QUERY_SET = ISpyBSafeQuerySet() +_ISPYB_SAFE_QUERY_SET = ISPyBSafeQuerySet() + + +class ValidateProjectMixin: + """Mixin for serializers to check if user is allowed to create objects. + + Requires a 'filter_permissions' member in the corresponding View. + This is used to navigate to the Project object from the data map + given to the validate() method. + """ + + def validate(self, data): + # User must be logged in + user = self.context['request'].user # type: ignore [attr-defined] + if not user or not user.is_authenticated: + raise serializers.ValidationError("You must be logged in") + view = self.context['view'] # type: ignore [attr-defined] + if not hasattr(view, "filter_permissions"): + raise AttributeError( + "The view object must define a 'filter_permissions' property" + ) + logger.info('view.filter_permissions=%s', view.filter_permissions) + + # We expect a filter_permissions string (defined in the View) like this... + # "compound__project_id" + # In this example the supplied data map is therefore expected to have a + # "compound" key (which we extract into a variable called 'base_object_key'). + # We use the 2nd half of the string (which we call 'project_path') + # to get to the Project object from 'data["compound"]'. + # + # If the filter_permissions string has no 2nd half (e.g. it's simply 'project_id') + # then the data is clearly expected to contain the Project object itself. + + base_object_key, project_path = view.filter_permissions.split('__', 1) + base_start_obj = data[base_object_key] + project_obj = ( + getattr(base_start_obj, project_path) if project_path else base_start_obj + ) + # Now get the proposals from the Project(s)... + if project_obj.__class__.__name__ == "ManyRelatedManager": + # Potential for many proposals... + object_proposals = [p.title for p in project_obj.all()] + else: + # Only one proposal... + object_proposals = [project_obj.title] + + # Now we have the proposals (Project titles) the object belongs to, + # has the user been associated (in IPSpyB) with any of them? + # We can always see (GET) objects that are open to the public. + restrict_public = False if self.context['request'].method == 'GET' else True # type: ignore [attr-defined] + if ( + object_proposals + and not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + user=user, + proposals=object_proposals, + restrict_public_to_membership=restrict_public, + ) + ): + raise PermissionDenied( + detail="Your authority to access this object has not been given" + ) + + # OK if we get here... + return data class FileSerializer(serializers.ModelSerializer): @@ -37,7 +101,7 @@ class Meta: fields = '__all__' -class CompoundIdentifierSerializer(serializers.ModelSerializer): +class CompoundIdentifierSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.CompoundIdentifier fields = '__all__' @@ -125,7 +189,6 @@ def get_metadata(self, obj): class Meta: model = models.Target - # TODO: it's missing protein_set. is it necessary anymore? fields = ( "id", "title", @@ -483,14 +546,24 @@ class Meta: # (POST, PUT, PATCH) -class SessionProjectWriteSerializer(serializers.ModelSerializer): +class SessionProjectWriteSerializer(ValidateProjectMixin, serializers.ModelSerializer): + # def validate_target(self, value): + # user = self.context['request'].user + # if not user or not user.is_authenticated: + # raise serializers.ValidationError("You must be logged in to create objects") + # if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target(user, value): + # raise serializers.ValidationError( + # "You have not been given access the object's Target" + # ) + # return value + class Meta: model = models.SessionProject fields = '__all__' # (GET, POST, PUT, PATCH) -class SessionActionsSerializer(serializers.ModelSerializer): +class SessionActionsSerializer(ValidateProjectMixin, serializers.ModelSerializer): actions = serializers.JSONField() class Meta: @@ -540,7 +613,7 @@ class Meta: # (GET, POST, PUT, PATCH) -class SnapshotActionsSerializer(serializers.ModelSerializer): +class SnapshotActionsSerializer(ValidateProjectMixin, serializers.ModelSerializer): actions = serializers.JSONField() class Meta: @@ -548,7 +621,7 @@ class Meta: fields = '__all__' -class ComputedSetSerializer(serializers.ModelSerializer): +class ComputedSetSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.ComputedSet fields = '__all__' @@ -667,7 +740,7 @@ class Meta: fields = '__all__' -class SiteObservationTagSerializer(serializers.ModelSerializer): +class SiteObservationTagSerializer(ValidateProjectMixin, serializers.ModelSerializer): site_observations = serializers.PrimaryKeyRelatedField( many=True, queryset=models.SiteObservation.objects.all() ) @@ -687,7 +760,7 @@ class Meta: } -class SessionProjectTagSerializer(serializers.ModelSerializer): +class SessionProjectTagSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.SessionProjectTag fields = '__all__' @@ -713,14 +786,14 @@ class DownloadStructuresSerializer(serializers.Serializer): # Start of Serializers for Squonk Jobs # (GET) -class JobFileTransferReadSerializer(serializers.ModelSerializer): +class JobFileTransferReadSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.JobFileTransfer fields = '__all__' # (POST, PUT, PATCH) -class JobFileTransferWriteSerializer(serializers.ModelSerializer): +class JobFileTransferWriteSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.JobFileTransfer fields = ("snapshot", "target", "squonk_project", "proteins", "compounds") @@ -768,7 +841,7 @@ class Meta: fields = ("job_status", "state_transition_time") -class TargetExperimentReadSerializer(serializers.ModelSerializer): +class TargetExperimentReadSerializer(ValidateProjectMixin, serializers.ModelSerializer): class Meta: model = models.ExperimentUpload fields = '__all__' @@ -845,7 +918,7 @@ class Meta: fields = '__all__' -class PoseSerializer(serializers.ModelSerializer): +class PoseSerializer(ValidateProjectMixin, serializers.ModelSerializer): site_observations = serializers.PrimaryKeyRelatedField( many=True, queryset=models.SiteObservation.objects.all(), @@ -921,7 +994,9 @@ def validate(self, data): - the pose they're being removed from is deleted when empty """ - logger.info('+ validate: %s', data) + logger.info('+ validate data: %s', data) + + data = super().validate(data) template = ( "Site observation {} cannot be assigned to pose because " diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index aadab6c0..7d4b6e92 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -21,7 +21,7 @@ from urllib3.exceptions import InsecureRequestWarning from wrapt import synchronized -from api.security import ISpyBSafeQuerySet +from api.security import ISPyBSafeQuerySet from viewer.models import Project, Squonk2Org, Squonk2Project, Squonk2Unit, Target, User _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -133,7 +133,7 @@ def __init__(self): # Used when we are given a tas (target access string). # It allows us to check that a user is permitted to use the access ID # and relies on ISPyB credentials present in the environment. - self.__ispyb_safe_query_set: ISpyBSafeQuerySet = ISpyBSafeQuerySet() + self.__ispyb_safe_query_set: ISPyBSafeQuerySet = ISPyBSafeQuerySet() def _get_user_name(self, user_id: int) -> str: # Gets the username (if id looks sensible) @@ -723,7 +723,7 @@ def _verify_access(self, c_params: CommonParams) -> Squonk2AgentRv: target_access_string = self._get_target_access_string(access_id) assert target_access_string proposal_list: List[str] = self.__ispyb_safe_query_set.get_proposals_for_user( - user, restrict_to_membership=True + user, restrict_public_to_membership=True ) if not target_access_string in proposal_list: msg = ( diff --git a/viewer/target_loader.py b/viewer/target_loader.py index 8a3cbb15..39a9617b 100644 --- a/viewer/target_loader.py +++ b/viewer/target_loader.py @@ -52,9 +52,6 @@ # assemblies and xtalforms XTALFORMS_FILE = "assemblies.yaml" -# canon site tag names -CANON_SITES_FILE = "canonical_sites.yaml" - # target name, nothing else CONFIG_FILE = "config*.yaml" @@ -136,6 +133,7 @@ def __str__(self): @dataclass class UploadReport: task: Task | None + proposal_ref: str stack: list[UploadReportEntry] = field(default_factory=list) upload_state: UploadState = UploadState.PROCESSING failed: bool = False @@ -173,6 +171,7 @@ def _update_task(self, message: str | list) -> None: self.task.update_state( state=self.upload_state, meta={ + "proposal_ref": self.proposal_ref, "description": message, }, ) @@ -464,7 +463,7 @@ def __init__( self.previous_version_dirs = None self.user_id = user_id - self.report = UploadReport(task=task) + self.report = UploadReport(task=task, proposal_ref=self.proposal_ref) self.raw_data.mkdir() @@ -1084,6 +1083,7 @@ def process_canon_site( Incoming data format: : + centroid_res: conformer_site_ids: global_reference_dtag: reference_conformer_site_id: @@ -1107,6 +1107,9 @@ def process_canon_site( ) residues = extract(key="residues", return_type=list) + centroid_res = extract(key="centroid_res") + conf_sites_ids = extract(key="conformer_site_ids", return_type=list) + ref_conf_site_id = extract(key="reference_conformer_site_id") fields = { "name": canon_site_id, @@ -1115,11 +1118,9 @@ def process_canon_site( defaults = { "residues": residues, + "centroid_res": centroid_res, } - conf_sites_ids = extract(key="conformer_site_ids", return_type=list) - ref_conf_site_id = extract(key="reference_conformer_site_id") - index_data = { "ref_conf_site": ref_conf_site_id, "conformer_site_ids": conf_sites_ids, @@ -1475,10 +1476,9 @@ def process_bundle(self): config = self._load_yaml(config_file) meta = self._load_yaml(Path(upload_dir).joinpath(METADATA_FILE)) xtalforms_yaml = self._load_yaml(Path(upload_dir).joinpath(XTALFORMS_FILE)) - canon_sites_yaml = self._load_yaml(Path(upload_dir).joinpath(CANON_SITES_FILE)) # this is the last file to load. if any of the files missing, don't continue - if not any([meta, config, xtalforms_yaml, canon_sites_yaml]): + if not any([meta, config, xtalforms_yaml]): msg = "Missing files in uploaded data, aborting" raise FileNotFoundError(msg) @@ -1661,24 +1661,6 @@ def process_bundle(self): canon_sites=canon_sites_by_conf_sites, xtalforms=xtalform_objects, ) - # enumerate xtalform_sites. a bit trickier than others because - # requires alphabetic enumeration - last_xtsite = ( - XtalformSite.objects.filter( - pk__in=[ - k.instance.pk - for k in xtalform_sites_objects.values() # pylint: disable=no-member - ] - ) - .order_by("-xtalform_site_num")[0] - .xtalform_site_num - ) - - xtnum = alphanumerator(start_from=last_xtsite) - for val in xtalform_sites_objects.values(): # pylint: disable=no-member - if not val.instance.xtalform_site_num: - val.instance.xtalform_site_num = next(xtnum) - val.instance.save() # now can update CanonSite with ref_conf_site # also, fill the canon_site_num field @@ -1811,16 +1793,12 @@ def process_bundle(self): logger.debug("data read and processed, adding tags") - canon_name_tag_map = { - k: v["centroid_res"] if "centroid_res" in v.keys() else "UNDEFINED" - for k, v in canon_sites_yaml.items() - } - # tag site observations cat_canon = TagCategory.objects.get(category="CanonSites") for val in canon_site_objects.values(): # pylint: disable=no-member prefix = val.instance.canon_site_num - tag = canon_name_tag_map.get(val.versioned_key, "UNDEFINED") + # tag = canon_name_tag_map.get(val.versioned_key, "UNDEFINED") + tag = val.versioned_key so_list = SiteObservation.objects.filter( canon_site_conf__canon_site=val.instance ) @@ -1839,12 +1817,10 @@ def process_bundle(self): f"{val.instance.canon_site.canon_site_num}" + f"{next(numerators[val.instance.canon_site.canon_site_num])}" ) - tag = val.instance.name.split('+')[0] + # tag = val.instance.name.split('+')[0] + tag = val.instance.name so_list = [ - site_observation_objects[k].instance - for k in val.index_data["members"] - # site_observations_versioned[k] - # for k in val.index_data["members"] + site_observation_objects[k].instance for k in val.index_data["members"] ] self._tag_observations( tag, prefix, category=cat_conf, site_observations=so_list, hidden=True @@ -1880,13 +1856,66 @@ def process_bundle(self): logger.debug("xtalform objects tagged") + # enumerate xtalform_sites. a bit trickier than others because + # requires alphabetic enumeration starting from the letter of + # the chain and following from there + + # sort the dictionary + # fmt: off + xtls_sort_qs = XtalformSite.objects.filter( + pk__in=[k.instance.pk for k in xtalform_sites_objects.values() ], # pylint: disable=no-member + ).annotate( + obvs=Count("canon_site__canonsiteconf__siteobservation", default=0), + ).order_by("-obvs", "xtalform_site_id") + # ordering by xtalform_site_id is not strictly necessary, but + # makes the sorting consistent + + # fmt: on + + _xtalform_sites_objects = {} + for xtl in xtls_sort_qs: + key = f"{xtl.xtalform_site_id}/{xtl.version}" + _xtalform_sites_objects[key] = xtalform_sites_objects[ + key + ] # pylint: disable=no-member + + if self.version_number == 1: + # first upload, use the chain letter + xtnum = alphanumerator( + start_from=xtls_sort_qs[0].lig_chain.lower(), drop_first=False + ) + else: + # subsequent upload, just use the latest letter as starting point + # fmt: off + last_xtsite = XtalformSite.objects.filter( + pk__in=[ + k.instance.pk + for k in _xtalform_sites_objects.values() # pylint: disable=no-member + ] + ).order_by( + "-xtalform_site_num" + )[0].xtalform_site_num + # fmt: on + xtnum = alphanumerator(start_from=last_xtsite) + + # this should be rare, as Frank said, all crystal-related + # issues should be resolved by the time of the first + # upload. In fact, I'll mark this momentous occasion here: + logger.warning("New XtalformSite objects added in subsequent uploads") + + for val in _xtalform_sites_objects.values(): # pylint: disable=no-member + if not val.instance.xtalform_site_num: + val.instance.xtalform_site_num = next(xtnum) + val.instance.save() + cat_xtalsite = TagCategory.objects.get(category="CrystalformSites") - for val in xtalform_sites_objects.values(): # pylint: disable=no-member + for val in _xtalform_sites_objects.values(): # pylint: disable=no-member prefix = ( f"F{val.instance.xtalform.xtalform_num}" + f"{val.instance.xtalform_site_num}" ) - tag = f"{val.instance.xtalform.name} - {val.instance.xtalform_site_id}" + # tag = val.instance.xtalform_site_id + tag = val.versioned_key so_list = [ site_observation_objects[k].instance for k in val.index_data["residues"] ] @@ -1966,7 +1995,9 @@ def _extract( def _generate_poses(self): values = ["canon_site_conf__canon_site", "cmpd"] # fmt: off - pose_groups = SiteObservation.objects.exclude( + pose_groups = SiteObservation.filter_manager.by_target( + self.target, + ).exclude( canon_site_conf__canon_site__isnull=True, ).exclude( cmpd__isnull=True, @@ -1999,16 +2030,23 @@ def _generate_poses(self): pose.save() except MultipleObjectsReturned: # must be a follow-up upload. create new pose, but - # only add observatons that are not yet assigned + # only add observatons that are not yet assigned (if + # these exist) pose_items = pose_items.filter(pose__isnull=True) - sample = pose_items.first() - pose = Pose( - canon_site=sample.canon_site_conf.canon_site, - compound=sample.cmpd, - main_site_observation=sample, - display_name=sample.code, - ) - pose.save() + if pose_items.exists(): + sample = pose_items.first() + pose = Pose( + canon_site=sample.canon_site_conf.canon_site, + compound=sample.cmpd, + main_site_observation=sample, + display_name=sample.code, + ) + pose.save() + else: + # I don't know if this can happen but this (due to + # other bugs) is what allowed me to find this + # error. Make a note in the logs. + logger.warning("No observations left to assign to pose") # finally add observations to the (new or existing) pose for obvs in pose_items: diff --git a/viewer/urls.py b/viewer/urls.py index f78a32d5..fe7e909f 100644 --- a/viewer/urls.py +++ b/viewer/urls.py @@ -6,7 +6,7 @@ urlpatterns = [ re_path(r"^react/*", views.react, name="react"), - path("upload_cset/", views.UploadCSet.as_view(), name="upload_cset"), + path("upload_cset/", views.UploadComputedSetView.as_view(), name="upload_cset"), path( "validate_task//", views.ValidateTaskView.as_view(), @@ -24,12 +24,14 @@ ), path("img_from_smiles/", views.img_from_smiles, name="img_from_smiles"), path("highlight_mol_diff/", views.highlight_mol_diff, name="highlight_mol_diff"), - path("sim_search/", views.similarity_search, name="sim_search"), path("open_targets/", views.get_open_targets, name="get_open_targets"), - path("compound_set//", views.cset_download, name="compound_set"), - path("protein_set//", views.pset_download, name="protein_set"), - path("upload_designs/", views.DSetUploadView.as_view(), name="upload_designs"), + path("compound_set//", views.computed_set_download, name="compound_set"), + path("upload_designs/", views.DesignSetUploadView.as_view(), name="upload_designs"), path("job_access/", views.JobAccessView.as_view(), name="job_access"), - path("task_status//", views.TaskStatus.as_view(), name="task_status"), - path("service_state/", views.ServiceState.as_view(), name="service_state"), + path( + "task_status//", + views.TaskStatusView.as_view(), + name="task_status", + ), + path("service_state/", views.ServiceStateView.as_view(), name="service_state"), ] diff --git a/viewer/utils.py b/viewer/utils.py index e170a0ee..8cf42987 100644 --- a/viewer/utils.py +++ b/viewer/utils.py @@ -1,8 +1,3 @@ -""" -utils.py - -Collection of technical methods tidied up in one location. -""" import fnmatch import itertools import json @@ -11,12 +6,17 @@ import shutil import string import tempfile +import uuid from pathlib import Path from typing import Dict, Generator, Optional from urllib.parse import urlparse +import pandas as pd from django.conf import settings from django.contrib.auth.models import User +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from django.core.mail import send_mail from django.db import IntegrityError, transaction from django.db.models import F from django.http import JsonResponse @@ -39,6 +39,12 @@ SDF_RECORD_SEPARATOR = '$$$$\n' +# The root of all files constructed by 'dicttocsv'. +# The directory must not contain anything but dicttocsv-generated files. +# It certainly must not be the root of the media directory or any other directory in it. +# Introduced during 1247 security review. +CSV_TO_DICT_DOWNLOAD_ROOT = os.path.join(settings.MEDIA_ROOT, 'downloads', 'dicttocsv') + def is_url(url: Optional[str]) -> bool: try: @@ -409,8 +415,17 @@ def restore_curated_tags(filename: str) -> None: logger.error(exc) -def alphanumerator(start_from: str = "") -> Generator[str, None, None]: - """Return alphabetic generator (A, B .. AA, AB...) starting from a specified point.""" +def alphanumerator( + start_from: str = "", drop_first: bool = True +) -> Generator[str, None, None]: + """Return alphabetic generator (A, B .. AA, AB...) starting from a specified point. + + drop_first - as per workflow, usually it's given the last letter + of previous sequence so the the next in the pipeline should be + start_from + 1. drop_first = False indicates this is not necessary + and start_from will be the first the iterator produces + + """ # since product requries finite maximum return string length set # to 10 characters. that should be enough for fragalysis (and to @@ -426,7 +441,90 @@ def alphanumerator(start_from: str = "") -> Generator[str, None, None]: if start_from is not None and start_from != '': start_from = start_from.lower() generator = itertools.dropwhile(lambda x: x != start_from, generator) # type: ignore[assignment] - # and drop one more, then it starts from after the start from as it should - _ = next(generator) + if drop_first: + # drop one more, then it starts from after the start from as it should + _ = next(generator) return generator + + +def save_tmp_file(myfile): + """Save file in temporary location for validation/upload processing""" + + name = myfile.name + path = default_storage.save('tmp/' + name, ContentFile(myfile.read())) + return str(os.path.join(settings.MEDIA_ROOT, path)) + + +def create_csv_from_dict(input_dict, title=None, filename=None): + """Write a CSV file containing data from an input dictionary and return a full + to the file (in the media directory). + """ + if not filename: + filename = 'download' + + unique_dir = str(uuid.uuid4()) + download_path = os.path.join(CSV_TO_DICT_DOWNLOAD_ROOT, unique_dir) + os.makedirs(download_path, exist_ok=True) + + download_file = os.path.join(download_path, filename) + + # Remove file if it already exists + if os.path.isfile(download_file): + os.remove(download_file) + + with open(download_file, "w", newline='', encoding='utf-8') as csvfile: + if title: + csvfile.write(title) + csvfile.write("\n") + + df = pd.DataFrame.from_dict(input_dict) + df.to_csv(download_file, mode='a', header=True, index=False) + + return download_file + + +def email_task_completion( + contact_email, message_type, target_name, target_path=None, task_id=None +): + """Notify user of upload completion""" + + logger.info('+ email_notify_task_completion: ' + message_type + ' ' + target_name) + email_from = settings.EMAIL_HOST_USER + + if contact_email == '' or not email_from: + # Only send email if configured. + return + + if message_type == 'upload-success': + subject = 'Fragalysis: Target: ' + target_name + ' Uploaded' + message = ( + 'The upload of your target data is complete. Your target is available at: ' + + str(target_path) + ) + elif message_type == 'validate-success': + subject = 'Fragalysis: Target: ' + target_name + ' Validation' + message = ( + 'Your data was validated. It can now be uploaded using the upload option.' + ) + else: + # Validation failure + subject = 'Fragalysis: Target: ' + target_name + ' Validation/Upload Failed' + message = ( + 'The validation/upload of your target data did not complete successfully. ' + 'Please navigate the following link to check the errors: validate_task/' + + str(task_id) + ) + + recipient_list = [ + contact_email, + ] + logger.info('+ email_notify_task_completion email_from: %s', email_from) + logger.info('+ email_notify_task_completion subject: %s', subject) + logger.info('+ email_notify_task_completion message: %s', message) + logger.info('+ email_notify_task_completion contact_email: %s', contact_email) + + # Send email - this should not prevent returning to the screen in the case of error. + send_mail(subject, message, email_from, recipient_list, fail_silently=True) + logger.info('- email_notify_task_completion') + return diff --git a/viewer/views.py b/viewer/views.py index f1b58e35..70897af3 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -3,11 +3,9 @@ import os import shlex import shutil -import uuid -import zipfile from datetime import datetime from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from wsgiref.util import FileWrapper import pandas as pd @@ -16,23 +14,18 @@ from celery.result import AsyncResult from dateutil.parser import parse from django.conf import settings -from django.core.files.base import ContentFile -from django.core.files.storage import default_storage -from django.core.mail import send_mail -from django.db import connections from django.http import FileResponse, HttpResponse, JsonResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.views import View -from rest_framework import mixins, permissions, status, viewsets -from rest_framework.exceptions import ParseError +from rest_framework import generics, mixins, permissions, status, viewsets from rest_framework.parsers import BaseParser from rest_framework.response import Response from rest_framework.views import APIView from api.infections import INFECTION_STRUCTURE_DOWNLOAD, have_infection -from api.security import ISpyBSafeQuerySet -from api.utils import get_highlighted_diffs, get_params, pretty_request +from api.security import ISPyBSafeQuerySet +from api.utils import get_highlighted_diffs, get_img_from_smiles, pretty_request from service_status.models import Service from viewer import filters, models, serializers from viewer.permissions import IsObjectProposalMember @@ -45,7 +38,13 @@ Squonk2AgentRv, get_squonk2_agent, ) -from viewer.utils import create_squonk_job_request_url, handle_uploaded_file +from viewer.utils import ( + CSV_TO_DICT_DOWNLOAD_ROOT, + create_csv_from_dict, + create_squonk_job_request_url, + handle_uploaded_file, + save_tmp_file, +) from .discourse import ( check_discourse_user, @@ -67,7 +66,6 @@ erase_compound_set_job_material, process_compound_set, process_compound_set_job_file, - process_design_sets, process_job_file_transfer, task_load_target, validate_compound_set, @@ -84,15 +82,20 @@ _SQ2A: Squonk2Agent = get_squonk2_agent() +_ISPYB_SAFE_QUERY_SET = ISPyBSafeQuerySet() + # --------------------------- # ENTRYPOINT FOR THE FRONTEND # --------------------------- def react(request): - """We "START HERE". This is the first API call that the front-end calls.""" - - discourse_api_key = settings.DISCOURSE_API_KEY + """ + The f/e starts here. + This is the first API call that the front-end calls, and it returns a 'context' + defining the state of things like the legacy URL, Squonk and Discourse availability + via the 'context' variable used for the view's template. + """ # Start building the context that will be passed to the template context = {'legacy_url': settings.LEGACY_URL} @@ -107,13 +110,14 @@ def react(request): logger.info("Squonk2 is NOT configured") context['squonk_available'] = 'false' + discourse_api_key = settings.DISCOURSE_API_KEY context['discourse_available'] = 'true' if discourse_api_key else 'false' user = request.user if user.is_authenticated: context['discourse_host'] = '' context['user_present_on_discourse'] = 'false' - # If user is authenticated and a discourse api key is available, then check discourse to - # see if user is set up and set up flag in context. + # If user is authenticated and a discourse api key is available, + # hen check discourse to see if user is set up and set up flag in context. if discourse_api_key: context['discourse_host'] = settings.DISCOURSE_HOST _, _, user_id = check_discourse_user(user) @@ -122,7 +126,7 @@ def react(request): # User is authenticated, so if Squonk can be called # return the Squonk UI URL - # so the f/e knows where to go. + # so the f/e knows where to go to find it. context['squonk_ui_url'] = '' if sq2_rv.success and check_squonk_active(request): context['squonk_ui_url'] = _SQ2A.get_ui_url() @@ -130,76 +134,106 @@ def react(request): return render(request, "viewer/react_temp.html", context) -def save_pdb_zip(pdb_file): - zf = zipfile.ZipFile(pdb_file) - zip_lst = zf.namelist() - zfile = {} - zfile_hashvals: Dict[str, str] = {} - print(zip_lst) - for filename in zip_lst: - # only handle pdb files - if filename.split('.')[-1] == 'pdb': - f = filename.split('/')[0] - save_path = os.path.join(settings.MEDIA_ROOT, 'tmp', f) - if default_storage.exists(f): - rand_str = uuid.uuid4().hex - pdb_path = default_storage.save( - save_path.replace('.pdb', f'-{rand_str}.pdb'), - ContentFile(zf.read(filename)), - ) - # Test if Protein object already exists - # code = filename.split('/')[-1].replace('.pdb', '') - # test_pdb_code = filename.split('/')[-1].replace('.pdb', '') - # test_prot_objs = Protein.objects.filter(code=test_pdb_code) - # - # if len(test_prot_objs) > 0: - # # make a unique pdb code as not to overwrite existing object - # rand_str = uuid.uuid4().hex - # test_pdb_code = f'{code}#{rand_str}' - # zfile_hashvals[code] = rand_str - # - # fn = test_pdb_code + '.pdb' - # - # pdb_path = default_storage.save('tmp/' + fn, - # ContentFile(zf.read(filename))) - else: - pdb_path = default_storage.save( - save_path, ContentFile(zf.read(filename)) - ) - test_pdb_code = pdb_path.split('/')[-1].replace('.pdb', '') - zfile[test_pdb_code] = pdb_path +# -------------------- +# FUNCTION-BASED VIEWS +# -------------------- - # Close the zip file - if zf: - zf.close() - return zfile, zfile_hashvals +def img_from_smiles(request): + """Generate a 2D molecule image for a given smiles string""" + if "smiles" in request.GET and (smiles := request.GET["smiles"]): + return get_img_from_smiles(smiles, request) + else: + return HttpResponse("Please insert SMILES") -def save_tmp_file(myfile): - """Save file in temporary location for validation/upload processing""" +def highlight_mol_diff(request): + """Generate a 2D molecule image highlighting the difference between a + reference and new molecule + """ + if 'ref_smiles' in request.GET: + return HttpResponse(get_highlighted_diffs(request)) + else: + return HttpResponse("Please insert smiles for reference and probe") + - name = myfile.name - path = default_storage.save('tmp/' + name, ContentFile(myfile.read())) - tmp_file = str(os.path.join(settings.MEDIA_ROOT, path)) +def get_open_targets(request): + """Return a list of all open targets (viewer/open_targets)""" + # Unused arguments + del request - return tmp_file + targets = models.Target.objects.all() + target_names = [] + target_ids = [] + open_proposals: set = _ISPYB_SAFE_QUERY_SET.get_open_proposals() + for t in targets: + for p in t.project_id.all(): + if p.title in open_proposals: + target_names.append(t.title) + target_ids.append(t.id) + break -class CompoundIdentifierTypeView(viewsets.ModelViewSet): + return HttpResponse( + json.dumps({'target_names': target_names, 'target_ids': target_ids}) + ) + + +def computed_set_download(request, name): + """View to download an SDF file of a ComputedSet by name + (viewer/compound_set/()). + """ + # Unused arguments + del request + + computed_set = models.ComputedSet.objects.get(name=name) + if not computed_set: + return HttpResponse(status=status.HTTP_404_NOT_FOUND) + # Is the computed set available to the user? + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + request.user, computed_set.target + ): + return HttpResponse( + "You are not a member of the CompoundSet's Target Proposal", + status=status.HTTP_403_FORBIDDEN, + ) + + written_filename = computed_set.written_sdf_filename + with open(written_filename, 'r', encoding='utf-8') as wf: + data = wf.read() + response = HttpResponse(content_type='text/plain') + response[ + 'Content-Disposition' + ] = f'attachment; filename={name}.sdf' # force browser to download file + response.write(data) + return response + + +# ----------------- +# CLASS-BASED VIEWS +# ----------------- + + +class CompoundIdentifierTypeView(viewsets.ReadOnlyModelViewSet): queryset = models.CompoundIdentifierType.objects.all() serializer_class = serializers.CompoundIdentifierTypeSerializer permission_classes = [permissions.IsAuthenticated] -class CompoundIdentifierView(viewsets.ModelViewSet): +class CompoundIdentifierView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): queryset = models.CompoundIdentifier.objects.all() serializer_class = serializers.CompoundIdentifierSerializer - permission_classes = [permissions.IsAuthenticated] + filter_permissions = "compound__project_id" + permission_classes = [IsObjectProposalMember] filterset_fields = ["type", "compound"] -class VectorsView(ISpyBSafeQuerySet): +class VectorsView(ISPyBSafeQuerySet): """Vectors (api/vector)""" queryset = models.SiteObservation.filter_manager.filter_qs() @@ -208,7 +242,7 @@ class VectorsView(ISpyBSafeQuerySet): filter_permissions = "experiment__experiment_upload__target__project_id" -class MolecularPropertiesView(ISpyBSafeQuerySet): +class MolecularPropertiesView(ISPyBSafeQuerySet): """Molecular properties (api/molprops)""" queryset = models.Compound.filter_manager.filter_qs() @@ -217,7 +251,7 @@ class MolecularPropertiesView(ISpyBSafeQuerySet): filter_permissions = "experiment__experiment_upload__target__project_id" -class GraphView(ISpyBSafeQuerySet): +class GraphView(ISPyBSafeQuerySet): """Graph (api/graph)""" queryset = models.SiteObservation.filter_manager.filter_qs() @@ -226,7 +260,7 @@ class GraphView(ISpyBSafeQuerySet): filter_permissions = "experiment__experiment_upload__target__project_id" -class MolImageView(ISpyBSafeQuerySet): +class MolImageView(ISPyBSafeQuerySet): """Molecule images (api/molimg)""" queryset = models.SiteObservation.objects.filter() @@ -235,7 +269,7 @@ class MolImageView(ISpyBSafeQuerySet): filter_permissions = "experiment__experiment_upload__target__project_id" -class CompoundImageView(ISpyBSafeQuerySet): +class CompoundImageView(ISPyBSafeQuerySet): """Compound images (api/cmpdimg)""" queryset = models.Compound.filter_manager.filter_qs() @@ -244,7 +278,7 @@ class CompoundImageView(ISpyBSafeQuerySet): filterset_class = filters.CmpdImgFilter -class ProteinMapInfoView(ISpyBSafeQuerySet): +class ProteinMapInfoView(ISPyBSafeQuerySet): """Protein map info (file) (api/protmap)""" queryset = models.SiteObservation.objects.all() @@ -257,7 +291,7 @@ class ProteinMapInfoView(ISpyBSafeQuerySet): ) -class ProteinPDBInfoView(ISpyBSafeQuerySet): +class ProteinPDBInfoView(ISPyBSafeQuerySet): """Protein apo pdb info (file) (api/protpdb)""" queryset = models.SiteObservation.objects.all() @@ -270,7 +304,7 @@ class ProteinPDBInfoView(ISpyBSafeQuerySet): ) -class ProteinPDBBoundInfoView(ISpyBSafeQuerySet): +class ProteinPDBBoundInfoView(ISPyBSafeQuerySet): """Protein bound pdb info (file) (api/protpdbbound)""" queryset = models.SiteObservation.filter_manager.filter_qs() @@ -283,7 +317,7 @@ class ProteinPDBBoundInfoView(ISpyBSafeQuerySet): ) -class ProjectView(ISpyBSafeQuerySet): +class ProjectView(ISPyBSafeQuerySet): """Projects (api/project)""" queryset = models.Project.objects.filter() @@ -292,7 +326,7 @@ class ProjectView(ISpyBSafeQuerySet): filter_permissions = "" -class TargetView(mixins.UpdateModelMixin, ISpyBSafeQuerySet): +class TargetView(mixins.UpdateModelMixin, ISPyBSafeQuerySet): queryset = models.Target.objects.filter() serializer_class = serializers.TargetSerializer filter_permissions = "project_id" @@ -320,7 +354,7 @@ def patch(self, request, pk): ) -class CompoundView(ISpyBSafeQuerySet): +class CompoundView(ISPyBSafeQuerySet): """Compounds (api/compounds)""" queryset = models.Compound.filter_manager.filter_qs() @@ -329,14 +363,14 @@ class CompoundView(ISpyBSafeQuerySet): filterset_class = filters.CompoundFilter -class UploadCSet(APIView): - """Render and control viewer/upload-cset.html - a page allowing upload of computed sets. Validation and +class UploadComputedSetView(generics.ListCreateAPIView): + """Render and control viewer/upload-cset.html - a page allowing upload of computed sets. Validation and upload tasks are defined in `viewer.compound_set_upload`, `viewer.sdf_check` and `viewer.tasks` and the task response handling is done by `viewer.views.ValidateTaskView` and `viewer.views.UploadTaskView` """ def get(self, request): - tag = '+ UploadCSet GET' + tag = '+ UploadComputedSetView GET' logger.info('%s', pretty_request(request, tag=tag)) logger.info('User=%s', str(request.user)) # logger.info('Auth=%s', str(request.auth)) @@ -373,7 +407,7 @@ def get(self, request): return render(request, 'viewer/upload-cset.html', context) def post(self, request): - tag = '+ UploadCSet POST' + tag = '+ UploadComputedSetView POST' logger.info('%s', pretty_request(request, tag=tag)) logger.info('User=%s', str(request.user)) # logger.info('Auth=%s', str(request.auth)) @@ -391,7 +425,7 @@ def post(self, request): user = self.request.user logger.info( - '+ UploadCSet POST user.id=%s choice="%s" target="%s" update_set="%s"', + '+ UploadComputedSetView POST user.id=%s choice="%s" target="%s" update_set="%s"', user.id, choice, target, @@ -408,7 +442,7 @@ def post(self, request): else: request.session[_SESSION_ERROR] = 'The set could not be found' logger.warning( - '- UploadCSet POST error_msg="%s"', + '- UploadComputedSetView POST error_msg="%s"', request.session[_SESSION_ERROR], ) return redirect('viewer:upload_cset') @@ -422,7 +456,7 @@ def post(self, request): ' you must provide a Target and SDF file' ) logger.warning( - '- UploadCSet POST error_msg="%s"', + '- UploadComputedSetView POST error_msg="%s"', request.session[_SESSION_ERROR], ) return redirect('viewer:upload_cset') @@ -432,7 +466,7 @@ def post(self, request): _SESSION_ERROR ] = 'To Delete you must select an existing set' logger.warning( - '- UploadCSet POST error_msg="%s"', + '- UploadComputedSetView POST error_msg="%s"', request.session[_SESSION_ERROR], ) return redirect('viewer:upload_cset') @@ -453,7 +487,7 @@ def post(self, request): # If so redirect... if _SESSION_ERROR in request.session: logger.warning( - '- UploadCSet POST error_msg="%s"', + '- UploadComputedSetView POST error_msg="%s"', request.session[_SESSION_ERROR], ) return redirect('viewer:upload_cset') @@ -469,15 +503,9 @@ def post(self, request): context['error_message'] = f'Unknown Target ({target})' return render(request, 'viewer/upload-cset.html', context) # What proposals is the user a member of? - ispyb_safe_query_set = ISpyBSafeQuerySet() - user_proposals = ispyb_safe_query_set.get_proposals_for_user( - user, restrict_to_membership=True - ) - user_is_member = any( - target_project.title in user_proposals - for target_project in target_record.project_id.all() - ) - if not user_is_member: + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + user, target_record + ): context[ 'error_message' ] = f"You cannot load compound sets for '{target}'. You are not a member of any of its Proposals" @@ -518,7 +546,7 @@ def post(self, request): task_id = task_validate.id task_status = task_validate.status logger.info( - '+ UploadCSet POST "Validate" task underway' + '+ UploadComputedSetView POST "Validate" task underway' ' (validate_task_id=%s (%s) validate_task_status=%s)', task_id, type(task_id), @@ -552,7 +580,7 @@ def post(self, request): task_id = task_upload.id task_status = task_upload.status logger.info( - '+ UploadCSet POST "Upload" task underway' + '+ UploadComputedSetView POST "Upload" task underway' ' (upload_task_id=%s (%s) upload_task_status=%s)', task_id, type(task_id), @@ -604,70 +632,27 @@ def post(self, request): _SESSION_MESSAGE ] = f'Compound set "{selected_set_name}" deleted' - logger.info('+ UploadCSet POST "Delete" done') + logger.info('+ UploadComputedSetView POST "Delete" done') return redirect('viewer:upload_cset') else: logger.warning( - '+ UploadCSet POST unsupported submit_choice value (%s)', choice + '+ UploadComputedSetView POST unsupported submit_choice value (%s)', + choice, ) else: - logger.warning('- UploadCSet POST form.is_valid() returned False') + logger.warning( + '- UploadComputedSetView POST form.is_valid() returned False' + ) - logger.info('- UploadCSet POST (leaving)') + logger.info('- UploadComputedSetView POST (leaving)') context = {'form': form} return render(request, 'viewer/upload-cset.html', context) -def email_task_completion( - contact_email, message_type, target_name, target_path=None, task_id=None -): - """Notify user of upload completion""" - - logger.info('+ email_notify_task_completion: ' + message_type + ' ' + target_name) - email_from = settings.EMAIL_HOST_USER - - if contact_email == '' or not email_from: - # Only send email if configured. - return - - if message_type == 'upload-success': - subject = 'Fragalysis: Target: ' + target_name + ' Uploaded' - message = ( - 'The upload of your target data is complete. Your target is available at: ' - + str(target_path) - ) - elif message_type == 'validate-success': - subject = 'Fragalysis: Target: ' + target_name + ' Validation' - message = ( - 'Your data was validated. It can now be uploaded using the upload option.' - ) - else: - # Validation failure - subject = 'Fragalysis: Target: ' + target_name + ' Validation/Upload Failed' - message = ( - 'The validation/upload of your target data did not complete successfully. ' - 'Please navigate the following link to check the errors: validate_task/' - + str(task_id) - ) - - recipient_list = [ - contact_email, - ] - logger.info('+ email_notify_task_completion email_from: %s', email_from) - logger.info('+ email_notify_task_completion subject: %s', subject) - logger.info('+ email_notify_task_completion message: %s', message) - logger.info('+ email_notify_task_completion contact_email: %s', contact_email) - - # Send email - this should not prevent returning to the screen in the case of error. - send_mail(subject, message, email_from, recipient_list, fail_silently=True) - logger.info('- email_notify_task_completion') - return - - class ValidateTaskView(View): """View to handle dynamic loading of validation results from `viewer.tasks.validate`. The validation of files uploaded to viewer/upload_cset. @@ -694,9 +679,6 @@ def get(self, request, validate_task_id): - html (str): html of task outcome - success message or html table of errors & fail message """ - # Unused arguments - del request - logger.info('+ ValidateTaskView.get') validate_task_id_str = str(validate_task_id) @@ -720,6 +702,28 @@ def get(self, request, validate_task_id): # Response from validation is a tuple validate_dict = results[1] validated = results[2] + # [3] comes from task in tasks.py, 4th element in task payload tuple + task_data = results[3] + + if isinstance(task_data, dict) and 'target' in task_data.keys(): + target_name = task_data['target'] + try: + target = models.Target.objects.get(title=target_name) + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + request.user, target + ): + return Response( + {'error': "You are not a member of the Target's proposal"}, + status=status.HTTP_403_FORBIDDEN, + ) + except models.Target.DoesNotExist: + # the name is filled from db, so this not existing would be extraordinary + logger.error('Target %s not found', target_name) + return Response( + {'error': f'Target {target_name} not found'}, + status=status.HTTP_403_FORBIDDEN, + ) + if validated: response_data[ 'html' @@ -802,9 +806,6 @@ def get(self, request, upload_task_id): - processed (str): 'None' - html (str): message to tell the user their data was not processed """ - # Unused arguments - del request - logger.debug('+ UploadTaskView.get') upload_task_id_str = str(upload_task_id) task = AsyncResult(upload_task_id_str) @@ -846,6 +847,15 @@ def get(self, request, upload_task_id): response_data['validated'] = 'Validated' cset_name = results[1] cset = models.ComputedSet.objects.get(name=cset_name) + + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target( + request.user, cset.target + ): + return Response( + {'error': "You are not a member of the Target's proposal"}, + status=status.HTTP_403_FORBIDDEN, + ) + name = cset.name response_data['results'] = {} response_data['results']['cset_download_url'] = ( @@ -867,125 +877,6 @@ def get(self, request, upload_task_id): return JsonResponse(response_data) -def img_from_smiles(request): - """Generate a 2D molecule image for a given smiles string""" - if "smiles" in request.GET: - smiles = request.GET["smiles"] - if smiles: - return get_params(smiles, request) - else: - return HttpResponse("Please insert SMILES") - else: - return HttpResponse("Please insert SMILES") - - -def highlight_mol_diff(request): - """Generate a 2D molecule image highlighting the difference between a - reference and new molecule - """ - if 'prb_smiles' and 'ref_smiles' in request.GET: - return HttpResponse(get_highlighted_diffs(request)) - else: - return HttpResponse("Please insert smiles for reference and probe") - - -def similarity_search(request): - if "smiles" in request.GET: - smiles = request.GET["smiles"] - else: - return HttpResponse("Please insert SMILES") - if "db_name" in request.GET: - db_name = request.GET["db_name"] - else: - return HttpResponse("Please insert db_name") - sql_query = """SELECT sub.* - FROM ( - SELECT rdk.id,rdk.structure,rdk.idnumber - FROM vendordbs.enamine_real_dsi_molfps AS mfp - JOIN vendordbs.enamine_real_dsi AS rdk ON mfp.id = rdk.id - WHERE m @> qmol_from_smiles(%s) LIMIT 1000 - ) sub;""" - with connections[db_name].cursor() as cursor: - cursor.execute(sql_query, [smiles]) - return HttpResponse(json.dumps(cursor.fetchall())) - - -def get_open_targets(request): - """Return a list of all open targets (viewer/open_targets)""" - # Unused arguments - del request - - targets = models.Target.objects.all() - target_names = [] - target_ids = [] - - for t in targets: - for p in t.project_id.all(): - if 'OPEN' in p.title: - target_names.append(t.title) - target_ids.append(t.id) - - return HttpResponse( - json.dumps({'target_names': target_names, 'target_ids': target_ids}) - ) - - -def cset_download(request, name): - """View to download an SDF file of a ComputedSet by name - (viewer/compound_set/()). - """ - # Unused arguments - del request - - computed_set = models.ComputedSet.objects.get(name=name) - written_filename = computed_set.written_sdf_filename - with open(written_filename, 'r', encoding='utf-8') as wf: - data = wf.read() - response = HttpResponse(content_type='text/plain') - response[ - 'Content-Disposition' - ] = f'attachment; filename={name}.sdf' # force browser to download file - response.write(data) - return response - - -def pset_download(request, name): - """View to download a zip file of all protein structures (apo) for a computed set - (viewer/compound_set/()) - """ - # Unused arguments - del request - - response = HttpResponse(content_type='application/zip') - filename = 'protein-set_' + name + '.zip' - response['Content-Disposition'] = ( - 'filename=%s' % filename - ) # force browser to download file - - # For the first stage (green release) of the XCA-based Fragalysis Stack - # there are no PDB files. - # compound_set = models.ComputedSet.objects.get(name=name) - # computed_molecules = models.ComputedMolecule.objects.filter(computed_set=compound_set) - # pdb_filepaths = list(set([c.pdb_info.path for c in computed_molecules])) - # buff = StringIO() - # zip_obj = zipfile.ZipFile(buff, 'w') - # zip_obj.writestr('') - # for fp in pdb_filepaths: - # data = open(fp, 'r', encoding='utf-8').read() - # zip_obj.writestr(fp.split('/')[-1], data) - # zip_obj.close() - # buff.flush() - # ret_zip = buff.getvalue() - # buff.close() - - # ...instead we just create an empty file... - with zipfile.ZipFile('dummy.zip', 'w') as pdb_file: - pass - - response.write(pdb_file) - return response - - # Start of ActionType class ActionTypeView(viewsets.ModelViewSet): """View to retrieve information about action types available to users (GET). @@ -1004,7 +895,12 @@ class ActionTypeView(viewsets.ModelViewSet): # Start of Session Project -class SessionProjectsView(viewsets.ModelViewSet): +class SessionProjectView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """View to retrieve information about user projects (collection of sessions) (GET). Also used for saving project information (PUT, POST, PATCH). (api/session-projects). @@ -1013,67 +909,69 @@ class SessionProjectsView(viewsets.ModelViewSet): queryset = models.SessionProject.objects.filter() filter_permissions = "target__project_id" filterset_fields = '__all__' + permission_classes = [IsObjectProposalMember] def get_serializer_class(self): - """Determine which serializer to use based on whether the request is a GET or a POST, PUT or PATCH request - - Returns - ------- - Serializer (rest_framework.serializers.ModelSerializer): - - if GET: `viewer.serializers.SessionProjectReadSerializer` - - if other: `viewer.serializers.SessionProjectWriteSerializer` - """ if self.request.method in ['GET']: - # GET return serializers.SessionProjectReadSerializer - # (POST, PUT, PATCH) return serializers.SessionProjectWriteSerializer -class SessionActionsView(viewsets.ModelViewSet): +class SessionActionsView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """View to retrieve information about actions relating to sessions_project (GET). Also used for saving project action information (PUT, POST, PATCH). (api/session-actions). """ - queryset = models.SessionActions.objects.filter() + queryset = models.SessionActions.filter_manager.filter_qs() + filter_permissions = "session_project__target__project_id" serializer_class = serializers.SessionActionsSerializer + permission_classes = [IsObjectProposalMember] # Note: jsonField for Actions will need specific queries - can introduce if needed. filterset_fields = ('id', 'author', 'session_project', 'last_update_date') -class SnapshotsView(viewsets.ModelViewSet): +class SnapshotView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """View to retrieve information about user sessions (snapshots) (GET). Also used for saving session information (PUT, POST, PATCH). (api/snapshots) """ - queryset = models.Snapshot.objects.filter() + queryset = models.Snapshot.filter_manager.filter_qs() + filter_permissions = "session_project__target__project_id" + filterset_class = filters.SnapshotFilter def get_serializer_class(self): - """Determine which serializer to use based on whether the request is a GET or a POST, PUT or PATCH request - - Returns - ------- - Serializer (rest_framework.serializers.ModelSerializer): - - if GET: `viewer.serializers.SnapshotReadSerializer` - - if other: `viewer.serializers.SnapshotWriteSerializer` - """ if self.request.method in ['GET']: return serializers.SnapshotReadSerializer return serializers.SnapshotWriteSerializer - filterset_class = filters.SnapshotFilter - -class SnapshotActionsView(viewsets.ModelViewSet): +class SnapshotActionsView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """View to retrieve information about actions relating to snapshots (GET). Also used for saving snapshot action information (PUT, POST, PATCH). (api/snapshot-actions). """ - queryset = models.SnapshotActions.objects.filter() + queryset = models.SnapshotActions.filter_manager.filter_qs() + filter_permissions = "snapshot__session_project__target__project_id" serializer_class = serializers.SnapshotActionsSerializer + permission_classes = [IsObjectProposalMember] # Note: jsonField for Actions will need specific queries - can introduce if needed. filterset_fields = ( @@ -1085,7 +983,7 @@ class SnapshotActionsView(viewsets.ModelViewSet): ) -class DSetCSVParser(BaseParser): +class DesignSetCSVParser(BaseParser): """ CSV parser class specific to design set csv spec - sets media_type for DSetUploadView to text/csv @@ -1094,60 +992,72 @@ class DSetCSVParser(BaseParser): media_type = 'text/csv' -class DSetUploadView(APIView): +class DesignSetUploadView(APIView): """Upload a design set (PUT) from a csv file""" - parser_class = (DSetCSVParser,) + parser_class = (DesignSetCSVParser,) def put(self, request, format=None): # pylint: disable=redefined-builtin """Method to handle PUT request and upload a design set""" # Don't need... - del format - - f = request.FILES['file'] - set_type = request.PUT['type'] - set_description = request.PUT['description'] - - # save uploaded file to temporary storage - name = f.name - path = default_storage.save('tmp/' + name, ContentFile(f.read())) - tmp_file = str(os.path.join(settings.MEDIA_ROOT, path)) - - df = pd.read_csv(tmp_file) - mandatory_cols = ['set_name', 'smiles', 'identifier', 'inspirations'] - actual_cols = df.columns - for col in mandatory_cols: - if col not in actual_cols: - raise ParseError( - "The 4 following columns are mandatory: set_name, smiles, identifier, inspirations" - ) + del format, request - set_names, compounds = process_design_sets(df, set_type, set_description) + # Unsupported for now, as part of 1247 (securing endpoints) + return HttpResponse(status=status.HTTP_404_NOT_FOUND) - string = 'Design set(s) successfully created: ' + # BEGIN removed as part of 1247 (securing endpoints) + # This code is unused by the f/e - length = len(set_names) - string += str(length) + '; ' - for i in range(0, length): - string += ( - str(i + 1) - + ' - ' - + set_names[i] - + ') number of compounds = ' - + str(len(compounds[i])) - + '; ' - ) + # f = request.FILES['file'] + # set_type = request.PUT['type'] + # set_description = request.PUT['description'] + + # # save uploaded file to temporary storage + # name = f.name + # path = default_storage.save('tmp/' + name, ContentFile(f.read())) + # tmp_file = str(os.path.join(settings.MEDIA_ROOT, path)) + + # df = pd.read_csv(tmp_file) + # mandatory_cols = ['set_name', 'smiles', 'identifier', 'inspirations'] + # actual_cols = df.columns + # for col in mandatory_cols: + # if col not in actual_cols: + # raise ParseError( + # "The 4 following columns are mandatory: set_name, smiles, identifier, inspirations" + # ) + + # set_names, compounds = process_design_sets(df, set_type, set_description) + + # string = 'Design set(s) successfully created: ' + + # length = len(set_names) + # string += str(length) + '; ' + # for i in range(0, length): + # string += ( + # str(i + 1) + # + ' - ' + # + set_names[i] + # + ') number of compounds = ' + # + str(len(compounds[i])) + # + '; ' + # ) - return HttpResponse(json.dumps(string)) + # return HttpResponse(json.dumps(string)) + # END removed as part of 1247 (securing endpoints) -class ComputedSetView(viewsets.ModelViewSet): + +class ComputedSetView( + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """Retrieve information about and delete computed sets.""" queryset = models.ComputedSet.objects.filter() serializer_class = serializers.ComputedSetSerializer - filter_permissions = "project_id" + filter_permissions = "target__project_id" filterset_fields = ('target', 'target__title') + permission_classes = [IsObjectProposalMember] http_method_names = ['get', 'head', 'delete'] @@ -1163,52 +1073,52 @@ def destroy(self, request, pk=None): return HttpResponse(status=204) -class ComputedMoleculesView(viewsets.ReadOnlyModelViewSet): +class ComputedMoleculesView(ISPyBSafeQuerySet): """Retrieve information about computed molecules - 3D info (api/compound-molecules).""" queryset = models.ComputedMolecule.objects.all() serializer_class = serializers.ComputedMoleculeSerializer - filter_permissions = "project_id" + filter_permissions = "compound__project_id" filterset_fields = ('computed_set',) -class NumericalScoresView(viewsets.ReadOnlyModelViewSet): +class NumericalScoreValuesView(ISPyBSafeQuerySet): """View to retrieve information about numerical computed molecule scores (api/numerical-scores). """ queryset = models.NumericalScoreValues.objects.all() serializer_class = serializers.NumericalScoreSerializer - filter_permissions = "project_id" + filter_permissions = "compound__compound__project_id" filterset_fields = ('compound', 'score') -class TextScoresView(viewsets.ReadOnlyModelViewSet): +class TextScoresView(ISPyBSafeQuerySet): """View to retrieve information about text computed molecule scores (api/text-scores).""" queryset = models.TextScoreValues.objects.all() serializer_class = serializers.TextScoreSerializer - filter_permissions = "project_id" + filter_permissions = "compound__compound__project_id" filterset_fields = ('compound', 'score') -class CompoundScoresView(viewsets.ReadOnlyModelViewSet): +class CompoundScoresView(ISPyBSafeQuerySet): """View to retrieve descriptions of scores for a given name or computed set.""" queryset = models.ScoreDescription.objects.all() serializer_class = serializers.ScoreDescriptionSerializer - filter_permissions = "project_id" + filter_permissions = "computed_set__target__project_id" filterset_fields = ('computed_set', 'name') -class ComputedMolAndScoreView(viewsets.ReadOnlyModelViewSet): +class ComputedMolAndScoreView(ISPyBSafeQuerySet): """View to retrieve all information about molecules from a computed set along with all of their scores. """ queryset = models.ComputedMolecule.objects.all() serializer_class = serializers.ComputedMolAndScoreSerializer - filter_permissions = "project_id" + filter_permissions = "compound__project_id" filterset_fields = ('computed_set',) @@ -1240,6 +1150,12 @@ class DiscoursePostView(viewsets.ViewSet): def create(self, request): """Method to handle POST request and call discourse to create the post""" logger.info('+ DiscoursePostView.post') + if not request.user.is_authenticated: + content: Dict[str, Any] = { + 'error': 'Only authenticated users can post content to Discourse' + } + return Response(content, status=status.HTTP_403_FORBIDDEN) + data = request.data logger.info('+ DiscoursePostView.post %s', json.dumps(data)) @@ -1296,77 +1212,70 @@ def list(self, request): return Response({"Posts": posts}) -def create_csv_from_dict(input_dict, title=None, filename=None): - """Write a CSV file containing data from an input dictionary and return a URL - to the file in the media directory. - """ - if not filename: - filename = 'download' - - media_root = settings.MEDIA_ROOT - unique_dir = str(uuid.uuid4()) - # /code/media/downloads/unique_dir - download_path = os.path.join(media_root, 'downloads', unique_dir) - os.makedirs(download_path, exist_ok=True) - - download_file = os.path.join(download_path, filename) - - # Remove file if it already exists - if os.path.isfile(download_file): - os.remove(download_file) - - with open(download_file, "w", newline='', encoding='utf-8') as csvfile: - if title: - csvfile.write(title) - csvfile.write("\n") - - df = pd.DataFrame.from_dict(input_dict) - df.to_csv(download_file, mode='a', header=True, index=False) - - return download_file - - -class DictToCsv(viewsets.ViewSet): +class DictToCSVView(viewsets.ViewSet): """Takes a dictionary and returns a download link to a CSV file with the data.""" serializer_class = serializers.DictToCsvSerializer def list(self, request): - """Method to handle GET request""" - file_url = request.GET.get('file_url') - - filename = str(Path(file_url).name) - - if file_url and os.path.isfile(file_url): - with open(file_url, encoding='utf8') as csvfile: - # return file and tidy up. - response = HttpResponse(csvfile, content_type='text/csv') - # response['Content-Disposition'] = 'attachment; filename=download.csv' - response['Content-Disposition'] = f'attachment; filename={filename}' - shutil.rmtree(os.path.dirname(file_url), ignore_errors=True) - return response - else: - return Response("Please provide file_url parameter") + """Method to handle GET request. + If the file exists it is returned and then removed.""" + file_url = request.GET.get('file_url', '') + logger.info('DictToCsv file_url="%s"', file_url) + + # The file is expected to include a full path + # to a file in the dicttocsv directory. + real_file_url = os.path.realpath(file_url) + if ( + os.path.commonpath([CSV_TO_DICT_DOWNLOAD_ROOT, real_file_url]) + != CSV_TO_DICT_DOWNLOAD_ROOT + ): + logger.warning( + 'DictToCsv path is invalid (file_url="%s" real_file_url="%s")', + file_url, + real_file_url, + ) + return Response("Please provide a file_url for an existing DictToCsv file") + elif not os.path.isfile(real_file_url): + logger.warning( + 'DictToCsv file does not exist (file_url="%s" real_file_url="%s")', + file_url, + real_file_url, + ) + return Response("The given DictToCsv file does not exist") + + with open(real_file_url, encoding='utf8') as csvfile: + # return file and tidy up. + response = HttpResponse(csvfile, content_type='text/csv') + # response['Content-Disposition'] = 'attachment; filename=download.csv' + filename = str(Path(real_file_url).name) + response['Content-Disposition'] = f'attachment; filename={filename}' + shutil.rmtree(os.path.dirname(real_file_url), ignore_errors=True) + return response def create(self, request): - """Method to handle POST request""" - logger.info('+ DictToCsv.post') + """Method to handle POST request. Creates a file that the user + is then expected to GET.""" input_dict = request.data['dict'] input_title = request.data['title'] filename = request.data.get('filename', 'download.csv') + logger.info('title="%s" input_dict size=%s', input_title, len(input_dict)) if not input_dict: - return Response({"message": "Please enter Dictionary"}) + return Response({"message": "Please provide a dictionary"}) else: - filename_url = create_csv_from_dict( - input_dict, input_title, filename=filename - ) + file_url = create_csv_from_dict(input_dict, input_title, filename=filename) + logger.info( + 'Created file_url="%s" (size=%s)', + file_url, + os.path.getsize(file_url), + ) - return Response({"file_url": filename_url}) + return Response({"file_url": file_url}) # Classes Relating to Tags -class TagCategoryView(viewsets.ModelViewSet): +class TagCategoryView(viewsets.ReadOnlyModelViewSet): """Set up and retrieve information about tag categories (api/tag_category).""" queryset = models.TagCategory.objects.all() @@ -1374,11 +1283,18 @@ class TagCategoryView(viewsets.ModelViewSet): filterset_fields = ('id', 'category') -class SiteObservationTagView(viewsets.ModelViewSet): +class SiteObservationTagView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """Set up/retrieve information about tags relating to Molecules (api/molecule_tag)""" queryset = models.SiteObservationTag.objects.all() + filter_permissions = "target__project_id" serializer_class = serializers.SiteObservationTagSerializer + permission_classes = [IsObjectProposalMember] filterset_fields = ( 'id', 'tag', @@ -1389,24 +1305,39 @@ class SiteObservationTagView(viewsets.ModelViewSet): ) -class PoseView(viewsets.ModelViewSet): +class PoseView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """Set up/retrieve information about Poses (api/poses)""" queryset = models.Pose.filter_manager.filter_qs() + filter_permissions = "compound__project_id" serializer_class = serializers.PoseSerializer filterset_class = filters.PoseFilter - http_method_names = ('get', 'post', 'patch') -class SessionProjectTagView(viewsets.ModelViewSet): +class SessionProjectTagView( + mixins.UpdateModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + ISPyBSafeQuerySet, +): """Set up/retrieve information about tags relating to Session Projects.""" queryset = models.SessionProjectTag.objects.all() + filter_permissions = "target__project_id" serializer_class = serializers.SessionProjectTagSerializer + permission_classes = [IsObjectProposalMember] filterset_fields = ('id', 'tag', 'category', 'target', 'session_projects') -class DownloadStructures(ISpyBSafeQuerySet): +class DownloadStructuresView( + mixins.CreateModelMixin, + ISPyBSafeQuerySet, +): """Uses a selected subset of the target data (proteins and booleans with suggested files) and creates a Zip file with the contents. @@ -1499,6 +1430,13 @@ def create(self, request): return Response(content, status=status.HTTP_404_NOT_FOUND) logger.info('Found Target record %r', target) + # Is the user part of the target's proposal? + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target(request.user, target): + msg = 'You have not been given access to this Target' + logger.warning(msg) + content = {'message': msg} + return Response(content, status=status.HTTP_403_FORBIDDEN) + proteins_list = [ p.strip() for p in request.data.get('proteins', '').split(',') if p ] @@ -1544,7 +1482,7 @@ def create(self, request): return Response({"file_url": filename_url}) -class UploadTargetExperiments(ISpyBSafeQuerySet): +class UploadExperimentUploadView(ISPyBSafeQuerySet): serializer_class = serializers.TargetExperimentWriteSerializer permission_class = [permissions.IsAuthenticated] http_method_names = ('post',) @@ -1572,7 +1510,7 @@ def create(self, request, *args, **kwargs): return redirect(settings.LOGIN_URL) else: if target_access_string not in self.get_proposals_for_user( - user, restrict_to_membership=True + user, restrict_public_to_membership=True ): return Response( { @@ -1612,16 +1550,22 @@ def create(self, request, *args, **kwargs): return Response({'task_status_url': url}, status=status.HTTP_202_ACCEPTED) -class TaskStatus(APIView): +class TaskStatusView(APIView): def get(self, request, task_id, *args, **kwargs): """Given a task_id (a string) we try to return the status of the task, trying to handle unknown tasks as best we can. """ # Unused arguments - del request, args, kwargs + del args, kwargs logger.debug("task_id=%s", task_id) + if not request.user.is_authenticated and settings.AUTHENTICATE_UPLOAD: + content: Dict[str, Any] = { + 'error': 'Only authenticated users can check the task status' + } + return Response(content, status=status.HTTP_403_FORBIDDEN) + # task_id is a UUID, but Celery expects a string task_id_str = str(task_id) result = None @@ -1640,9 +1584,26 @@ def get(self, request, task_id, *args, **kwargs): messages = [] if hasattr(result, 'info'): if isinstance(result.info, dict): + # check if user is allowed to view task info + proposal = result.info.get('proposal_ref', '') + + if proposal not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user( + request.user + ): + return Response( + {'error': 'You are not a member of the proposal f"proposal"'}, + status=status.HTTP_403_FORBIDDEN, + ) + messages = result.info.get('description', []) elif isinstance(result.info, list): - messages = result.info + # this path should never materialize + logger.error('result.info attribute list instead of dict') + return Response( + {'error': 'You are not a member of the proposal f"proposal"'}, + status=status.HTTP_403_FORBIDDEN, + ) + # messages = result.info started = result.state != 'PENDING' finished = result.ready() @@ -1663,7 +1624,7 @@ def get(self, request, task_id, *args, **kwargs): return JsonResponse(data) -class DownloadTargetExperiments(viewsets.ModelViewSet): +class DownloadExperimentUploadView(viewsets.ModelViewSet): serializer_class = serializers.TargetExperimentDownloadSerializer permission_class = [permissions.IsAuthenticated] http_method_names = ('post',) @@ -1675,17 +1636,64 @@ def create(self, request, *args, **kwargs): # Unused arguments del args, kwargs - logger.info("+ DownloadTargetExperiments.create called") + logger.info("+ DownloadExperimentUploadView.create called") serializer = self.get_serializer_class()(data=request.data) if serializer.is_valid(): - # project = serializer.validated_data['project'] - # target = serializer.validated_data['target'] - filename = serializer.validated_data['filename'] + # To permit a download the user must be a member of the target's proposal + # (or the proposal must be open) + project: models.Project = serializer.validated_data['project'] + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + request.user, [project.title], restrict_public_to_membership=False + ): + return Response( + {'error': "You have no access to the Project"}, + status=status.HTTP_403_FORBIDDEN, + ) + target: models.Target = serializer.validated_data['target'] + if project not in target.project_id.all(): + return Response( + {'error': "The Target is not part of the given Project"}, + status=status.HTTP_403_FORBIDDEN, + ) - # source_dir = Path(settings.MEDIA_ROOT).joinpath(TARGET_LOADER_DATA) - source_dir = Path(settings.MEDIA_ROOT).joinpath('tmp') - file_path = source_dir.joinpath(filename) + # Now we have to search for an ExperimentUpload that matches the Target + # and the filename combination. + filename = serializer.validated_data['filename'] + exp_uploads: List[ + models.ExperimentUpload + ] = models.ExperimentUpload.objects.filter( + target=target, + file=filename, + ) + if len(exp_uploads) > 1: + return Response( + { + 'error': "More than one ExperimentUpload matches your Target and Filename" + }, + status=status.HTTP_400_INTERNAL_SERVER_ERROR, + ) + elif len(exp_uploads) == 0: + return Response( + {'error': "No ExperimentUpload matches your Target and Filename"}, + status=status.HTTP_404_NOT_FOUND, + ) + # Use the only experiment upload found. + # We don't need the user's filename any more, + # it's embedded in the ExperimentUpload's file field. + exp_upload: models.ExperimentUpload = exp_uploads[0] + file_path = exp_upload.get_download_path() + logger.info( + "Found exp_upload=%s file_path=%s", + exp_upload, + file_path, + ) + if not file_path.exists(): + return Response( + {'error': f"TargetExperiment file '{filename}' not found"}, + status=status.HTTP_404_NOT_FOUND, + ) + # Finally, return the file wrapper = FileWrapper(open(file_path, 'rb')) response = FileResponse(wrapper, content_type='application/zip') response['Content-Disposition'] = ( @@ -1693,10 +1701,11 @@ def create(self, request, *args, **kwargs): ) response['Content-Length'] = os.path.getsize(file_path) return response + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) -class TargetExperimentUploads(ISpyBSafeQuerySet): +class ExperimentUploadView(ISPyBSafeQuerySet): queryset = models.ExperimentUpload.objects.all() serializer_class = serializers.TargetExperimentReadSerializer permission_class = [permissions.IsAuthenticated] @@ -1705,39 +1714,36 @@ class TargetExperimentUploads(ISpyBSafeQuerySet): http_method_names = ('get',) -class SiteObservations(viewsets.ModelViewSet): +class SiteObservationView(ISPyBSafeQuerySet): queryset = models.SiteObservation.filter_manager.filter_qs().filter( superseded=False ) serializer_class = serializers.SiteObservationReadSerializer - permission_class = [permissions.IsAuthenticated] filterset_class = filters.SiteObservationFilter - filter_permissions = "target__project_id" - http_method_names = ('get',) + filter_permissions = "experiment__experiment_upload__project" -class CanonSites(viewsets.ModelViewSet): +class CanonSiteView(ISPyBSafeQuerySet): queryset = models.CanonSite.filter_manager.filter_qs().filter(superseded=False) serializer_class = serializers.CanonSiteReadSerializer - permission_class = [permissions.IsAuthenticated] filterset_class = filters.CanonSiteFilter - http_method_names = ('get',) + filter_permissions = ( + "ref_conf_site__ref_site_observation__experiment__experiment_upload__project" + ) -class CanonSiteConfs(viewsets.ModelViewSet): +class CanonSiteConfView(ISPyBSafeQuerySet): queryset = models.CanonSiteConf.filter_manager.filter_qs().filter(superseded=False) serializer_class = serializers.CanonSiteConfReadSerializer filterset_class = filters.CanonSiteConfFilter - permission_class = [permissions.IsAuthenticated] - http_method_names = ('get',) + filter_permissions = "ref_site_observation__experiment__experiment_upload__project" -class XtalformSites(viewsets.ModelViewSet): +class XtalformSiteView(ISPyBSafeQuerySet): queryset = models.XtalformSite.filter_manager.filter_qs().filter(superseded=False) serializer_class = serializers.XtalformSiteReadSerializer filterset_class = filters.XtalformSiteFilter - permission_class = [permissions.IsAuthenticated] - http_method_names = ('get',) + filter_permissions = "canon_site__ref_conf_site__ref_site_observation__experiment__experiment_upload__project" class JobFileTransferView(viewsets.ModelViewSet): @@ -1762,7 +1768,7 @@ def get_serializer_class(self): def create(self, request): """Method to handle POST request""" - logger.info('+ JobFileTransfer.post') + logger.info('+ JobFileTransferView.post') # Only authenticated users can transfer files to sqonk user = self.request.user if not user.is_authenticated: @@ -1943,7 +1949,7 @@ def get_serializer_class(self): return serializers.JobOverrideWriteSerializer def create(self, request): - logger.info('+ JobOverride.post') + logger.info('+ JobOverrideView.post') # Only authenticated users can transfer files to sqonk user = self.request.user if not user.is_authenticated: @@ -1951,6 +1957,9 @@ def create(self, request): 'error': 'Only authenticated users can provide Job overrides' } return Response(content, status=status.HTTP_403_FORBIDDEN) + if not user.is_staff: + content = {'error': 'Only STAFF (Admin) users can provide Job overrides'} + return Response(content, status=status.HTTP_403_FORBIDDEN) # Override is expected to be a JSON string, # but protect against format issues @@ -1984,7 +1993,7 @@ def create(self, request): class JobRequestView(APIView): def get(self, request): - logger.info('+ JobRequest.get') + logger.info('+ JobRequestView.get') user = self.request.user if not user.is_authenticated: @@ -2007,16 +2016,22 @@ def get(self, request): snapshot_id = request.query_params.get('snapshot', None) if snapshot_id: - logger.info('+ JobRequest.get snapshot_id=%s', snapshot_id) + logger.info('+ JobRequestView.get snapshot_id=%s', snapshot_id) job_requests = models.JobRequest.objects.filter(snapshot=int(snapshot_id)) else: - logger.info('+ JobRequest.get snapshot_id=(unset)') + logger.info('+ JobRequestView.get snapshot_id=(unset)') job_requests = models.JobRequest.objects.all() for jr in job_requests: + # Skip any JobRequests the user does not have access to + if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + user, [jr.project.title] + ): + continue + # An opportunity to update JobRequest timestamps? if not jr.job_has_finished(): logger.info( - '+ JobRequest.get (id=%s) has not finished (job_status=%s)', + '+ JobRequestView.get (id=%s) has not finished (job_status=%s)', jr.id, jr.job_status, ) @@ -2025,7 +2040,7 @@ def get(self, request): # To get the current status. To do this we'll need # the 'callback context' we supplied when launching the Job. logger.info( - '+ JobRequest.get (id=%s, code=%s) getting update from Squonk...', + '+ JobRequestView.get (id=%s, code=%s) getting update from Squonk...', jr.id, jr.code, ) @@ -2035,14 +2050,14 @@ def get(self, request): # 'LOST', 'SUCCESS' or 'FAILURE' if not sq2a_rv.success: logger.warning( - '+ JobRequest.get (id=%s, code=%s) check failed (%s)', + '+ JobRequestView.get (id=%s, code=%s) check failed (%s)', jr.id, jr.code, sq2a_rv.msg, ) elif sq2a_rv.success and sq2a_rv.msg: logger.info( - '+ JobRequest.get (id=%s, code=%s) new status is (%s)', + '+ JobRequestView.get (id=%s, code=%s) new status is (%s)', jr.id, jr.code, sq2a_rv.msg, @@ -2057,7 +2072,7 @@ def get(self, request): jr.save() else: logger.info( - '+ JobRequest.get (id=%s, code=%s) is (probably) still running', + '+ JobRequestView.get (id=%s, code=%s) is (probably) still running', jr.id, jr.code, ) @@ -2066,7 +2081,7 @@ def get(self, request): results.append(serializer.data) num_results = len(results) - logger.info('+ JobRequest.get num_results=%s', num_results) + logger.info('+ JobRequestView.get num_results=%s', num_results) # Simulate the original paged API response... content = { @@ -2078,7 +2093,7 @@ def get(self, request): return Response(content, status=status.HTTP_200_OK) def post(self, request): - logger.info('+ JobRequest.post') + logger.info('+ JobRequestView.post') # Only authenticated users can create squonk job requests # (unless 'AUTHENTICATE_UPLOAD' is False in settings.py) user = self.request.user @@ -2113,14 +2128,14 @@ def post(self, request): return Response(content, status=status.HTTP_404_NOT_FOUND) # The user must be a member of the target access string. # (when AUTHENTICATE_UPLOAD is set) - if settings.AUTHENTICATE_UPLOAD: - ispyb_safe_query_set = ISpyBSafeQuerySet() - user_proposals = ispyb_safe_query_set.get_proposals_for_user( - user, restrict_to_membership=True + if ( + settings.AUTHENTICATE_UPLOAD + and not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals( + user, [project.title] ) - if project.title not in user_proposals: - content = {'error': f"You are not a member of '{project.title}'"} - return Response(content, status=status.HTTP_403_FORBIDDEN) + ): + content = {'error': f"You are not a member of '{project.title}'"} + return Response(content, status=status.HTTP_403_FORBIDDEN) # Check the user can use this Squonk2 facility. # To do this we need to setup a couple of API parameter objects. @@ -2485,7 +2500,7 @@ def get(self, request): return Response(ok_response) -class ServiceState(View): +class ServiceStateView(View): def get(self, *args, **kwargs): """Poll external service status. diff --git a/xcdb/__init__.py b/xcdb/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/xcdb/schema.py b/xcdb/schema.py deleted file mode 100644 index 3ae0bf02..00000000 --- a/xcdb/schema.py +++ /dev/null @@ -1,117 +0,0 @@ -import graphene -from graphene_django.rest_framework.mutation import SerializerMutation -from xchem_db.serializers import ( - CompoundsSerializer, - CrystalSerializer, - DataProcessingSerializer, - DimpleSerializer, - FragspectCrystalSerializer, - LabSerializer, - PanddaAnalysisSerializer, - PanddaEventSerializer, - PanddaRunSerializer, - PanddaSiteSerializer, - ProasisOutSerializer, - ReferenceSerializer, - RefinementSerializer, - SoakdbFilesSerializer, - TargetSerializer, -) - -relay = graphene.relay - - -class Target(SerializerMutation): - serializer_class = TargetSerializer - interfaces = (relay.Node,) - - -class Compounds(SerializerMutation): - serializer_class = CompoundsSerializer - interfaces = (relay.Node,) - - -class Reference(SerializerMutation): - serializer_class = ReferenceSerializer - interfaces = (relay.Node,) - - -class SoakdbFiles(SerializerMutation): - serializer_class = SoakdbFilesSerializer - interfaces = (relay.Node,) - - -class Crystal(SerializerMutation): - serializer_class = CrystalSerializer - interfaces = (relay.Node,) - - -class DataProcessing(SerializerMutation): - serializer_class = DataProcessingSerializer - interfaces = (relay.Node,) - - -class Dimple(SerializerMutation): - serializer_class = DimpleSerializer - interfaces = (relay.Node,) - - -class Lab(SerializerMutation): - serializer_class = LabSerializer - interfaces = (relay.Node,) - - -class Refinement(SerializerMutation): - serializer_class = RefinementSerializer - interfaces = (relay.Node,) - - -class PanddaAnalysis(SerializerMutation): - serializer_class = PanddaAnalysisSerializer - interfaces = (relay.Node,) - - -class PanddaRun(SerializerMutation): - serializer_class = PanddaRunSerializer - interfaces = (relay.Node,) - - -class PanddaSite(SerializerMutation): - serializer_class = PanddaSiteSerializer - interfaces = (relay.Node,) - - -class PanddaEvent(SerializerMutation): - serializer_class = PanddaEventSerializer - interfaces = (relay.Node,) - - -class ProasisOut(SerializerMutation): - serializer_class = ProasisOutSerializer - interfaces = (relay.Node,) - - -class Fragspect(SerializerMutation): - serializer_class = FragspectCrystalSerializer - interfaces = (relay.Node,) - - -class Query(graphene.ObjectType): - target = graphene.list(Target) - compounds = graphene.list(Compounds) - reference = graphene.list(Reference) - soakdb_files = graphene.list(SoakdbFiles) - crystal = graphene.list(Crystal) - data_processing = graphene.list(DataProcessing) - dimple = graphene.list(Dimple) - lab = graphene.list(Lab) - refinement = graphene.list(Refinement) - pandda_analysis = graphene.list(PanddaAnalysis) - pandda_run = graphene.list(PanddaRun) - pandda_site = graphene.list(PanddaSite) - pandda_event = graphene.list(PanddaEvent) - proasis_out = graphene.list(ProasisOut) - fragspect = graphene.list(Fragspect) - - -schema = graphene.Schema(query=Query) diff --git a/xcdb/urls.py b/xcdb/urls.py deleted file mode 100644 index 878be92b..00000000 --- a/xcdb/urls.py +++ /dev/null @@ -1,33 +0,0 @@ -from django.conf.urls import include -from django.urls import path -from rest_framework.authtoken import views as drf_views -from rest_framework.routers import DefaultRouter - -from xcdb.views import ( - CrystalView, - DataProcessingView, - DimpleView, - FragspectCrystalView, - LabView, - PanddaEventStatsView, - PanddaEventView, - ProasisOutView, - RefinementView, -) - -router = DefaultRouter() - -router.register(r'crystal', CrystalView) -router.register(r'dataproc', DataProcessingView) -router.register(r'dimple', DimpleView) -router.register(r'lab', LabView) -router.register(r'refinement', RefinementView) -router.register(r'pandda_event', PanddaEventView) -router.register(r'pandda_event_stats', PanddaEventStatsView) -router.register(r'proasis_out', ProasisOutView) -router.register(r'fragspect', FragspectCrystalView) - -urlpatterns = [ - path("", include(router.urls)), - path("auth", drf_views.obtain_auth_token, name="auth"), -] diff --git a/xcdb/views.py b/xcdb/views.py deleted file mode 100644 index 9982c092..00000000 --- a/xcdb/views.py +++ /dev/null @@ -1,182 +0,0 @@ -from xchem_db.models import ( - Crystal, - DataProcessing, - Dimple, - Lab, - PanddaEvent, - PanddaEventStats, - ProasisOut, - Refinement, -) -from xchem_db.serializers import ( - CrystalSerializer, - DataProcessingSerializer, - DimpleSerializer, - FragspectCrystalSerializer, - LabSerializer, - PanddaEventSerializer, - PanddaEventStatsSerializer, - ProasisOutSerializer, - RefinementSerializer, -) - -from api.security import ISpyBSafeQuerySet - - -class CrystalView(ISpyBSafeQuerySet): - queryset = Crystal.objects.filter() - filter_permissions = "visit__proposal" - serializer_class = CrystalSerializer - filter_fields = ( - "crystal_name", - "target__target_name", - "compound__smiles", - "visit__filename", - "visit__proposal__proposal", - "visit__visit", - ) - - -class DataProcessingView(ISpyBSafeQuerySet): - queryset = DataProcessing.objects.filter() - filter_permissions = "crystal_name__visit__proposal" - serializer_class = DataProcessingSerializer - filter_fields = ( - "crystal_name__crystal_name", - "crystal_name__target__target_name", - "crystal_name__compound__smiles", - "crystal_name__visit__filename", - "crystal_name__visit__proposal__proposal", - "crystal_name__visit__visit", - ) - - -class DimpleView(ISpyBSafeQuerySet): - queryset = Dimple.objects.filter() - filter_permissions = "crystal_name__visit__proposal" - serializer_class = DimpleSerializer - filter_fields = ( - "crystal_name__crystal_name", - "crystal_name__target__target_name", - "crystal_name__compound__smiles", - "crystal_name__visit__filename", - "crystal_name__visit__proposal__proposal", - "crystal_name__visit__visit", - "reference__reference_pdb", - ) - - -class LabView(ISpyBSafeQuerySet): - queryset = Lab.objects.filter() - filter_permissions = "crystal_name__visit__proposal" - serializer_class = LabSerializer - filter_fields = ( - "crystal_name__crystal_name", - "crystal_name__target__target_name", - "crystal_name__compound__smiles", - "crystal_name__visit__filename", - "crystal_name__visit__proposal__proposal", - "crystal_name__visit__visit", - "data_collection_visit", - "library_name", - "library_plate", - ) - - -class RefinementView(ISpyBSafeQuerySet): - queryset = Refinement.objects.filter() - filter_permissions = "crystal_name__visit__proposal" - serializer_class = RefinementSerializer - filter_fields = ( - "crystal_name__crystal_name", - "crystal_name__target__target_name", - "crystal_name__compound__smiles", - "crystal_name__visit__filename", - "crystal_name__visit__proposal__proposal", - "crystal_name__visit__visit", - "outcome", - ) - - -class PanddaEventView(ISpyBSafeQuerySet): - queryset = PanddaEvent.objects.filter() - filter_permissions = "crystal__visit__proposal" - serializer_class = PanddaEventSerializer - filter_fields = ( - "crystal__crystal_name", - "crystal__target__target_name", - "crystal__compound__smiles", - "crystal__visit__filename", - "crystal__visit__proposal__proposal", - "crystal__visit__visit", - # "pandda_run__pandda_analysis__pandda_dir", - # "pandda_run__pandda_log", - # "pandda_run__sites_file", - # "pandda_run__events_file", - # "pandda_run__input_dir", - # "site__site", - # "event", - # "lig_id", - # "pandda_event_map_native", - # "pandda_model_pdb", - # "pandda_input_mtz", - # "pandda_input_pdb", - ) - - -class PanddaEventStatsView(ISpyBSafeQuerySet): - queryset = PanddaEventStats.objects.filter() - filter_permissions = 'event__crystal__visit__proposal' - serializer_class = PanddaEventStatsSerializer - filter_fields = ( - "event__crystal__crystal_name", - "event__crystal__target__target_name", - "event__crystal__compound__smiles", - "event__crystal__visit__filename", - "event__crystal__visit__proposal__proposal", - "event__crystal__visit__visit", - ) - - -class ProasisOutView(ISpyBSafeQuerySet): - queryset = ProasisOut.objects.filter() - filter_permissions = "crystal__visit__proposal" - serializer_class = ProasisOutSerializer - filter_fields = ( - "crystal__crystal_name", - "crystal__target__target_name", - "crystal__compound__smiles", - "crystal__visit__filename", - "crystal__visit__proposal__proposal", - "crystal__visit__visit", - "proasis__strucid", - "proasis__crystal_name__crystal_name", - "proasis__crystal_name__target__target_name", - "proasis__crystal_name__compound__smiles", - "proasis__crystal_name__visit__filename", - "proasis__crystal_name__visit__proposal__proposal", - "proasis__crystal_name__visit__visit", - "proasis__refinement__crystal_name__crystal_name", - "proasis__refinement__crystal_name__target__target_name", - "proasis__refinement__crystal_name__compound__smiles", - "proasis__refinement__crystal_name__visit__filename", - "proasis__refinement__crystal_name__visit__proposal__proposal", - "proasis__refinement__crystal_name__visit__visit", - "proasis__refinement__outcome", - "root", - "start", - ) - - -class FragspectCrystalView(ISpyBSafeQuerySet): - queryset = PanddaEvent.objects.filter().prefetch_related( - 'crystal__target', - 'crystal__compound', - 'crystal', - 'site', - 'refinement', - 'data_proc', - ) - serializer_class = FragspectCrystalSerializer - filter_fields = {'crystal__target__target_name': ['iexact']} - filter_permissions = "crystal__visit__proposal"