views.py

import json
import logging
import os
import shutil
import zipfile
from datetime import datetime, timezone
from io import BytesIO
from pathlib import Path
from typing import Any, Dict, List, Optional
from wsgiref.util import FileWrapper

import pandas as pd
import pytz
from celery import Celery
from celery.result import AsyncResult
from dateutil.parser import parse
from django.conf import settings
from django.contrib.auth import get_user_model
from django.http import FileResponse, HttpResponse, JsonResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.views import View
from rest_framework import generics, mixins, permissions, status, viewsets
from rest_framework.decorators import action
from rest_framework.parsers import BaseParser
from rest_framework.response import Response
from rest_framework.views import APIView

from api.infections import INFECTION_STRUCTURE_DOWNLOAD, have_infection
from api.security import ISPyBSafeQuerySet
from api.utils import get_highlighted_diffs, get_img_from_smiles, pretty_request
from service_status.models import Service
from viewer import filters, models, serializers
from viewer.permissions import IsObjectProposalMember
from viewer.squonk2_agent import (
    AccessParams,
    CommonParams,
    RunJobParams,
    SendParams,
    Squonk2Agent,
    Squonk2AgentRv,
    get_squonk2_agent,
)
from viewer.target_loader import split_version, validate_data_version
from viewer.utils import (
    CSV_TO_DICT_DOWNLOAD_ROOT,
    create_csv_from_dict,
    create_squonk_job_request_url,
    handle_uploaded_file,
    save_tmp_file,
)

from .discourse import (
    check_discourse_user,
    create_discourse_post,
    list_discourse_posts_for_topic,
)
from .download_structures import (
    create_or_return_download_link,
    erase_out_of_date_download_records,
)
from .forms import CSetForm
from .squonk_job_file_transfer import validate_file_transfer_files
from .squonk_job_request import (
    check_squonk_active,
    create_squonk_job,
    get_squonk_job_config,
)
from .squonk_job_success_handler import job_success_handler
from .tags import load_tags_from_file
from .tasks import (
    process_compound_set,
    process_job_file_transfer,
    task_load_target,
    validate_compound_set,
)

logger = logging.getLogger(__name__)

# Fields injected in a session object to pass
# messages between views. This is used by UploadCSet
# to pass errors and other messages back to the user
# via the upload-cset.html template.
_SESSION_ERROR = 'session_error'
_SESSION_MESSAGE = 'session_message'

_SQ2A: Squonk2Agent = get_squonk2_agent()

_ISPYB_SAFE_QUERY_SET = ISPyBSafeQuerySet()

# ---------------------------
# ENTRYPOINT FOR THE FRONTEND
# ---------------------------


def react(request):
    """
    The f/e starts here.
    This is the first API call that the front-end calls, and it returns a 'context'
    defining the state of things like the legacy URL, Squonk and Discourse availability
    via the 'context' variable used for the view's template.
    """

    # ----
    # NOTE: If you add or remove any context keys here
    # ----  you MUST update the template that gets rendered
    #       (viewer/react_temp.html) so that is matches
    #       the keys you are creating and passing here.

    # Start building the context that will be passed to the template
    context = {'legacy_url': settings.LEGACY_URL}

    # Is the Squonk2 Agent configured?
    logger.info("Checking whether Squonk2 is configured...")
    sq2_rv = _SQ2A.configured()
    if sq2_rv.success:
        logger.info("Squonk2 is configured")
        context['squonk_available'] = 'true'
    else:
        logger.info("Squonk2 is NOT configured")
        context['squonk_available'] = 'false'

    discourse_api_key = settings.DISCOURSE_API_KEY
    context['discourse_available'] = 'true' if discourse_api_key else 'false'
    user = request.user
    if user.is_authenticated:
        context['discourse_host'] = ''
        context['user_present_on_discourse'] = 'false'
        # If user is authenticated and a discourse api key is available,
        # hen check discourse to see if user is set up and set up flag in context.
        if discourse_api_key:
            context['discourse_host'] = settings.DISCOURSE_HOST
            _, _, user_id = check_discourse_user(user)
            if user_id:
                context['user_present_on_discourse'] = 'true'

        # User is authenticated, so if Squonk can be called
        # return the Squonk UI URL
        # so the f/e knows where to go to find it.
        context['squonk_ui_url'] = ''
        if sq2_rv.success and check_squonk_active(request):
            context['squonk_ui_url'] = _SQ2A.get_ui_url()

    context['target_warning_message'] = settings.TARGET_WARNING_MESSAGE

    render_template = "viewer/react_temp.html"
    logger.info("Rendering %s with context=%s...", render_template, context)
    return render(request, render_template, context)


# --------------------
# FUNCTION-BASED VIEWS
# --------------------


def img_from_smiles(request):
    """Generate a 2D molecule image for a given smiles string"""
    if "smiles" in request.GET and (smiles := request.GET["smiles"]):
        return get_img_from_smiles(smiles, request)
    else:
        return HttpResponse("Please insert SMILES")


def highlight_mol_diff(request):
    """Generate a 2D molecule image highlighting the difference between a
    reference and new molecule
    """
    if 'ref_smiles' in request.GET:
        return HttpResponse(get_highlighted_diffs(request))
    else:
        return HttpResponse("Please insert smiles for reference and probe")


def get_open_targets(request):
    """Return a list of all open targets (viewer/open_targets)"""
    # Unused arguments
    del request

    targets = models.Target.objects.all()
    target_names = []
    target_ids = []

    open_proposals: set = _ISPYB_SAFE_QUERY_SET.get_open_proposals()
    for t in targets:
        if t.project.title in open_proposals:
            target_names.append(t.title)
            target_ids.append(t.id)
            break

    return HttpResponse(
        json.dumps({'target_names': target_names, 'target_ids': target_ids})
    )


# -----------------
# CLASS-BASED VIEWS
# -----------------


class CompoundIdentifierTypeView(viewsets.ReadOnlyModelViewSet):
    queryset = models.CompoundIdentifierType.objects.all()
    serializer_class = serializers.CompoundIdentifierTypeSerializer
    permission_classes = [permissions.IsAuthenticated]


class CompoundIdentifierView(
    mixins.UpdateModelMixin,
    mixins.CreateModelMixin,
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    queryset = models.CompoundIdentifier.objects.all()
    serializer_class = serializers.CompoundIdentifierSerializer
    filter_permissions = "compound__project_id"
    permission_classes = [IsObjectProposalMember]
    filterset_fields = ["type", "compound"]


class VectorsView(ISPyBSafeQuerySet):
    """Vectors (api/vector)"""

    queryset = models.SiteObservation.filter_manager.filter_qs()
    serializer_class = serializers.VectorsSerializer
    filterset_class = filters.VectorFilter
    filter_permissions = "experiment__experiment_upload__target__project"


class MolecularPropertiesView(ISPyBSafeQuerySet):
    """Molecular properties (api/molprops)"""

    queryset = models.Compound.filter_manager.filter_qs()
    serializer_class = serializers.MolpropsSerializer
    filterset_class = filters.MolpropsFilter
    filter_permissions = "experiment__experiment_upload__target__project"


class GraphView(ISPyBSafeQuerySet):
    """Graph (api/graph)"""

    queryset = models.SiteObservation.filter_manager.filter_qs()
    serializer_class = serializers.GraphSerializer
    filterset_class = filters.GraphFilter
    filter_permissions = "experiment__experiment_upload__target__project"


class MolImageView(ISPyBSafeQuerySet):
    """Molecule images (api/molimg)"""

    queryset = models.SiteObservation.objects.filter()
    serializer_class = serializers.MolImageSerializer
    filterset_class = filters.MolImgFilter
    filter_permissions = "experiment__experiment_upload__target__project"


class CompoundImageView(ISPyBSafeQuerySet):
    """Compound images (api/cmpdimg)"""

    queryset = models.Compound.filter_manager.filter_qs()
    serializer_class = serializers.CmpdImageSerializer
    filter_permissions = "project_id"
    filterset_class = filters.CmpdImgFilter


class ProteinMapInfoView(ISPyBSafeQuerySet):
    """Protein map info (file) (api/protmap)"""

    queryset = models.SiteObservation.objects.all()
    serializer_class = serializers.ProtMapInfoSerializer
    filter_permissions = "experiment__experiment_upload__target__project"
    filterset_fields = (
        "code",
        "experiment__experiment_upload__target",
        "experiment__experiment_upload__target__title",
    )


class ProteinPDBInfoView(ISPyBSafeQuerySet):
    """Protein apo pdb info (file) (api/protpdb)"""

    queryset = models.SiteObservation.objects.all()
    serializer_class = serializers.ProtPDBInfoSerializer
    filter_permissions = "experiment__experiment_upload__target__project"
    filterset_fields = (
        "code",
        "experiment__experiment_upload__target",
        "experiment__experiment_upload__target__title",
    )


class ProteinPDBBoundInfoView(ISPyBSafeQuerySet):
    """Protein bound pdb info (file) (api/protpdbbound)"""

    queryset = models.SiteObservation.filter_manager.filter_qs()
    serializer_class = serializers.ProtPDBBoundInfoSerializer
    filter_permissions = "experiment__experiment_upload__target__project"
    filterset_fields = (
        "code",
        "experiment__experiment_upload__target",
        "experiment__experiment_upload__target__title",
    )


class ProjectView(ISPyBSafeQuerySet):
    """Projects (api/project)"""

    queryset = models.Project.objects.filter()
    serializer_class = serializers.ProjectSerializer
    # Special case - Project filter permissions is blank.
    filter_permissions = ""


class TargetView(mixins.UpdateModelMixin, ISPyBSafeQuerySet):
    queryset = models.Target.objects.filter()
    serializer_class = serializers.TargetSerializer
    filter_permissions = "project"
    filterset_fields = ("id", "title", "project")
    permission_classes = [IsObjectProposalMember]

    def patch(self, request, pk):
        try:
            target = self.queryset.get(pk=pk)
        except models.Target.DoesNotExist:
            msg = f"Target pk={pk} does not exist"
            logger.warning(msg)
            return Response(
                {"message": msg},
                status=status.HTTP_404_NOT_FOUND,
            )

        serializer = self.serializer_class(target, data=request.data, partial=True)
        if serializer.is_valid():
            _ = serializer.save()
            return Response(serializer.data, status=status.HTTP_200_OK)
        else:
            return Response(
                {"message": "wrong parameters"}, status=status.HTTP_400_BAD_REQUEST
            )


class CompoundView(mixins.UpdateModelMixin, ISPyBSafeQuerySet):
    """Compounds (api/compounds)"""

    queryset = models.Compound.filter_manager.filter_qs()
    serializer_class = serializers.CompoundSerializer
    filter_permissions = "project_id"
    filterset_class = filters.CompoundFilter


class UploadComputedSetView(generics.ListCreateAPIView):
    """Render and control viewer/upload-cset.html - a page allowing upload of computed sets. Validation and
    upload tasks are defined in `viewer.compound_set_upload`, `viewer.sdf_check` and `viewer.tasks` and the task
    response handling is done by `viewer.views.ValidateTaskView` and `viewer.views.UploadTaskView`
    """

    def get(self, request):
        tag = '+ UploadComputedSetView GET'
        logger.info('%s', pretty_request(request, tag=tag))
        logger.info('User=%s', str(request.user))
        #        logger.info('Auth=%s', str(request.auth))

        # Any messages passed to us via the session?
        # Maybe from a redirect?
        # It so take them and remove them.
        session_error = None
        if _SESSION_ERROR in request.session:
            session_error = request.session[_SESSION_ERROR]
            del request.session[_SESSION_ERROR]
        session_message = None
        if _SESSION_MESSAGE in request.session:
            session_message = request.session[_SESSION_MESSAGE]
            del request.session[_SESSION_MESSAGE]

        # Only authenticated users can upload files
        # - this can be switched off in settings.py.
        user = self.request.user
        if not user.is_authenticated and settings.AUTHENTICATE_UPLOAD:
            context: Dict[str, Any] = {
                'error_message': 'Only authenticated users can upload files - please navigate to landing page and Login'
            }
            return render(request, 'viewer/upload-cset.html', context)

        form = CSetForm()
        existing_sets = models.ComputedSet.objects.all()
        context = {
            'form': form,
            'sets': existing_sets,
            _SESSION_ERROR: session_error,
            _SESSION_MESSAGE: session_message,
        }
        return render(request, 'viewer/upload-cset.html', context)

    def post(self, request):
        tag = '+ UploadComputedSetView POST'
        logger.info('%s', pretty_request(request, tag=tag))
        logger.info('User=%s', str(request.user))
        #        logger.info('Auth=%s', str(request.auth))

        form = CSetForm(request.POST, request.FILES)
        if form.is_valid():
            # Get all the variables needed from the form.
            # The fields we use will be based on the 'submit_choice',
            # expected to be one of V (validate), U (upload) or D delete
            choice = request.POST['submit_choice']
            # The 'sdf_file' and 'target_name' are only required for upload/update
            sdf_file = request.FILES.get('sdf_file')
            target_name = request.POST.get('target_name')
            proposal_ref = request.POST.get('proposal_ref')
            update_set = request.POST.get('update_set')

            user = self.request.user
            logger.info(
                '+ UploadComputedSetView POST user.id=%s choice="%s" target="%s" update_set="%s"',
                user.id,
                choice,
                target_name,
                update_set,
            )

            # If a set is named the ComputedSet cannot be 'Anonymous'
            # and the user has to be the owner.
            selected_set: Optional[models.ComputedSet] = None
            if update_set and update_set != 'None':
                try:
                    selected_set = models.ComputedSet.objects.get(pk=update_set)
                except models.ComputedSet.DoesNotExist:
                    request.session[_SESSION_ERROR] = 'The set could not be found'
                    logger.warning(
                        '- UploadComputedSetView POST error_msg="%s"',
                        request.session[_SESSION_ERROR],
                    )
                    return redirect('viewer:upload_cset')

            # If validating or uploading we need a Target, and SDF file.
            # If updating or deleting we need an update set (that's not 'None')
            if choice in ['V', 'U']:
                if sdf_file is None or target_name is None or not proposal_ref:
                    err_msg = (
                        'To Validate or Upload'
                        + ' you must provide a Target, Proposal ref. and SDF file'
                    )
                    request.session[_SESSION_ERROR] = err_msg
                    logger.warning(
                        '- UploadComputedSetView POST error_msg="%s"',
                        err_msg,
                    )
                    return redirect('viewer:upload_cset')
            elif choice in ['D']:
                if update_set == 'None':
                    request.session[
                        _SESSION_ERROR
                    ] = 'To Delete you must select an existing set'
                    logger.warning(
                        '- UploadComputedSetView POST error_msg="%s"',
                        request.session[_SESSION_ERROR],
                    )
                    return redirect('viewer:upload_cset')

            try:
                project = models.Project.objects.get(title=proposal_ref)
            except models.Project.DoesNotExist:
                request.session[_SESSION_ERROR] = f'Proposal {proposal_ref} not found'
                logger.warning(
                    '- UploadComputedSetView POST error_msg="%s"',
                    request.session[_SESSION_ERROR],
                )
                return redirect('viewer:upload_cset')

            if (
                settings.AUTHENTICATE_UPLOAD
                and project.title
                not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(request.user)
            ):
                context = {
                    'error_message': f"You cannot load compound sets for '{target_name}'. You are not a member of any of its Proposals"
                }
                return render(request, 'viewer/upload-cset.html', context)

            try:
                target = models.Target.objects.get(title=target_name, project=project)
            except models.Target.DoesNotExist:
                request.session[_SESSION_ERROR] = f'Target {target_name} not found'
                logger.warning(
                    '- UploadComputedSetView POST error_msg="%s"',
                    request.session[_SESSION_ERROR],
                )
                return redirect('viewer:upload_cset')

            # If uploading (updating) or deleting
            # the set owner cannot be anonymous
            # and the user needs to be the owner
            if choice in ['U', 'D'] and selected_set:
                if selected_set.owner_user.id == settings.ANONYMOUS_USER:
                    request.session[
                        _SESSION_ERROR
                    ] = 'You cannot Update or Delete Anonymous sets'
                elif selected_set.owner_user != user:
                    request.session[
                        _SESSION_ERROR
                    ] = 'You can only Update or Delete sets you own'
                # Something wrong?
                # If so redirect...
                if _SESSION_ERROR in request.session:
                    logger.warning(
                        '- UploadComputedSetView POST error_msg="%s"',
                        request.session[_SESSION_ERROR],
                    )
                    return redirect('viewer:upload_cset')

            # Save uploaded sdf and zip to tmp storage
            tmp_pdb_file = None
            tmp_sdf_file = None
            if 'pdb_zip' in list(request.FILES.keys()):
                #     # In the first stage (green release) of the XCA-based Fragalysis Stack
                #     # we do not support PDB files.
                #     request.session[
                #         _SESSION_ERROR
                #     ] = 'This release does not support the inclusion of PDB file.'
                #     logger.warning(
                #         '- UploadCSet POST error_msg="%s"', request.session[_SESSION_ERROR]
                #     )
                #     return redirect('viewer:upload_cset')
                pdb_file = request.FILES['pdb_zip']
                tmp_pdb_file = save_tmp_file(pdb_file)
            if sdf_file:
                tmp_sdf_file = save_tmp_file(sdf_file)

            if choice == 'V':
                # Validate
                # Start celery task
                task_params = {
                    'user_id': user.id,
                    'sdf_file': tmp_sdf_file,
                    'target': target.pk,
                }
                if tmp_pdb_file:
                    task_params['zfile'] = tmp_pdb_file
                if update_set:
                    task_params['update'] = update_set
                task_validate = validate_compound_set.delay(task_params)

                task_id = task_validate.id
                task_status = task_validate.status
                logger.info(
                    '+ UploadComputedSetView POST "Validate" task underway'
                    ' (validate_task_id=%s (%s) validate_task_status=%s)',
                    task_id,
                    type(task_id),
                    task_status,
                )

                # Update client side with task id and status
                context = {
                    'validate_task_id': task_id,
                    'validate_task_status': task_status,
                }
                return render(request, 'viewer/upload-cset.html', context)

            elif choice == 'U':
                # Upload
                # Start chained celery tasks. NB first function passes tuple
                # to second function - see tasks.py
                task_params = {
                    'user_id': user.id,
                    'sdf_file': tmp_sdf_file,
                    'target': target.pk,
                }
                if tmp_pdb_file:
                    task_params['zfile'] = tmp_pdb_file
                if update_set:
                    task_params['update'] = update_set
                task_upload = (
                    validate_compound_set.s(task_params) | process_compound_set.s()
                ).apply_async()

                task_id = task_upload.id
                task_status = task_upload.status
                logger.info(
                    '+ UploadComputedSetView POST "Upload" task underway'
                    ' (upload_task_id=%s (%s) upload_task_status=%s)',
                    task_id,
                    type(task_id),
                    task_status,
                )

                # Update client side with task id and status
                context = {'upload_task_id': task_id, 'upload_task_status': task_status}
                return render(request, 'viewer/upload-cset.html', context)

            elif choice == 'D':
                # Delete the object
                assert selected_set
                written_sdf_filename = selected_set.written_sdf_filename
                selected_set_name = selected_set.name

                # related objects:
                # - ComputedSetComputedMolecule
                # - ComputedMolecule
                # - NumericalScoreValues
                # - TextScoreValues
                # - ComputedMolecule_computed_inspirations
                # - Compound

                # all but ComputedMolecule are handled automatically
                # but (because of the m2m), have to delete those
                # separately

                # select ComputedMolecule objects that are in this set
                # and not in any other sets
                # fmt: off
                selected_set.computed_molecules.exclude(
                    pk__in=models.ComputedMolecule.objects.filter(
                        computed_set__in=models.ComputedSet.objects.filter(
                            target=selected_set.target,
                        ).exclude(
                            pk=selected_set.pk,
                        ),
                    ),
                ).delete()
                # fmt: on
                selected_set.delete()

                # ...and the original (expected) file
                if os.path.isfile(written_sdf_filename):
                    os.remove(written_sdf_filename)

                request.session[
                    _SESSION_MESSAGE
                ] = f'Compound set "{selected_set_name}" deleted'

                logger.info('+ UploadComputedSetView POST "Delete" done')

                return redirect('viewer:upload_cset')

            else:
                logger.warning(
                    '+ UploadComputedSetView POST unsupported submit_choice value (%s)',
                    choice,
                )

        else:
            logger.warning(
                '- UploadComputedSetView POST form.is_valid() returned False'
            )

        logger.info('- UploadComputedSetView POST (leaving)')

        context = {'form': form}
        return render(request, 'viewer/upload-cset.html', context)


class ValidateTaskView(View):
    """View to handle dynamic loading of validation results from `viewer.tasks.validate`.
    The validation of files uploaded to viewer/upload_cset.
    """

    def get(self, request, validate_task_id):
        """Get method for `ValidateTaskView`. Takes a validate task id, checks its
        status and returns the status, and result if the task is complete

        Returns
        -------
        response_data: JSON
            response data (dict) in JSON format:
                - if status = 'RUNNING':
                    - validate_task_status (str): task.status
                    - validate_task_id (str): task.id
                - if status = 'FAILURE':
                    - validate_task_status (str): task.status
                    - validate_task_id (str): task.id
                    - validate_traceback (str): task.traceback
                - if status = 'SUCCESS':
                    - validate_task_status (str): task.status
                    - validate_task_id (str): task.id
                    - html (str): html of task outcome - success message or html table of errors & fail message

        """
        logger.info('+ ValidateTaskView.get')
        validate_task_id_str = str(validate_task_id)

        task = AsyncResult(validate_task_id_str)
        response_data = {
            'validate_task_status': task.status,
            'validate_task_id': task.id,
        }

        if task.status == 'FAILURE':
            logger.info('+ ValidateTaskView.get.FAILURE')
            result = task.traceback
            response_data['validate_traceback'] = str(result)

            return JsonResponse(response_data)

        # Check if results ready
        if task.status == "SUCCESS":
            logger.info('+ ValidateTaskView.get.SUCCESS')
            results = task.get()
            # Response from validation is a tuple
            validate_dict = results[1]
            validated = results[2]
            # [3] comes from task in tasks.py, 4th element in task payload tuple
            task_data = results[3]

            if isinstance(task_data, dict) and 'proposal_ref' in task_data.keys():
                project_name = task_data['proposal_ref']
                try:
                    project = models.Project.objects.get(title=project_name)

                except models.Project.DoesNotExist:
                    logger.error('Project %s not found', project_name)
                    return Response(
                        {'error': f'Project {project_name} not found'},
                        status=status.HTTP_403_FORBIDDEN,
                    )

                if project.title not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(
                    request.user
                ):
                    return Response(
                        {
                            'error': f"You are not a member of the proposal {project_name}"
                        },
                        status=status.HTTP_403_FORBIDDEN,
                    )

            if validated:
                response_data[
                    'html'
                ] = 'Your data was validated. \n It can now be uploaded using the upload option.'
                response_data['validated'] = 'Validated'
                return JsonResponse(response_data)

            if not validated:
                # set pandas options to display all column data
                pd.set_option('display.max_colwidth', -1)

                table = pd.DataFrame.from_dict(validate_dict)
                html_table = table.to_html()
                html_table += '''<p> Your data was <b>not</b> validated. The table above shows errors</p>'''

                response_data["html"] = html_table
                response_data['validated'] = 'Not validated'

                return JsonResponse(response_data)

        return JsonResponse(response_data)


class UpdateTaskView(View):
    def get(self, request, update_task_id):
        # Unused arguments
        del request

        update_task_id_str = str(update_task_id)
        task = AsyncResult(update_task_id_str)
        response_data = {'update_task_status': task.status, 'update_task_id': task.id}

        result = 'Running...'

        if task.status == 'FAILURE':
            result = task.traceback
            response_data['result'] = str(result)

        if task.status == 'SUCCESS':
            result = task.get()

        response_data['result'] = str(result)

        return JsonResponse(response_data)


class UploadTaskView(View):
    """View to handle dynamic loading of upload results from `viewer.tasks.process_compound_set`.
    The upload of files for a computed set by a user at viewer/upload_cset.
    """

    def get(self, request, upload_task_id):
        """Get method for `UploadTaskView`. Takes an upload task id, checks its
        status and returns the status, and result if the task is complete

        Returns
        -------
        response_data: JSON
            response data (dict) in JSON format:
                - if status = 'RUNNING':
                    - upload_task_status (str): task.status
                    - upload_task_id (str): task.id
                - if status = 'FAILURE':
                    - upload_task_status (str): task.status
                    - upload_task_id (str): task.id
                    - upload_traceback (str): task.traceback
                - if status = 'SUCCESS':
                    - upload_task_status (str): task.status
                    - upload_task_id (str): task.id
                    - if results are a list (data was processed - validated or uploaded):
                        if this was a validation process
                        - validated (str): 'Not validated'
                        - html (str): html table of validation errors
                        if results are a validation/upload process:
                        - validated (str): 'Validated'
                        - results (dict): results
                        - results['cset_download_url'] (str): download url for computed set sdf file
                        - results['pset_download_url'] (str): download url for computed set pdb files (zip)
                    - if results are not string or list:
                        - processed (str): 'None'
                        - html (str): message to tell the user their data was not processed
        """
        logger.debug('+ UploadTaskView.get')
        upload_task_id_str = str(upload_task_id)
        task = AsyncResult(upload_task_id_str)
        response_data = {'upload_task_status': task.status, 'upload_task_id': task.id}

        if task.status == 'FAILURE':
            result = task.traceback
            response_data['upload_traceback'] = str(result)

            return JsonResponse(response_data)

        logger.debug('+ UploadTaskView.get() task.status=%s', task.status)
        if task.status == 'SUCCESS':
            results = task.get()
            logger.debug(
                '+ UploadTaskView.get() SUCCESS task.get()=%s (%s)',
                results,
                type(results),
            )

            # Was the task about validation or processing (an actual upload)?
            # We receive a list with the first value being 'validate' or 'process'
            if isinstance(results, list):
                if results[0] == 'validate':
                    # Get dictionary results
                    validate_dict = results[1]

                    # set pandas options to display all column data
                    pd.set_option('display.max_colwidth', -1)
                    table = pd.DataFrame.from_dict(validate_dict)
                    html_table = table.to_html()
                    html_table += '''<p> Your data was <b>not</b> validated. The table above shows errors</p>'''
                    response_data['validated'] = 'Not validated'
                    response_data['html'] = html_table

                    return JsonResponse(response_data)

                elif results[0] == 'process':
                    # section added to pass warnings to user after successful processing
                    logger.debug('processed warnings: %s', results[2])
                    # Upload/Update output tasks send back a tuple
                    response_data['results'] = {}
                    response_data['validated'] = 'Validated'
                    cset_id = int(results[1])
                    cset = models.ComputedSet.objects.get(pk=cset_id)

                    if (
                        settings.AUTHENTICATE_UPLOAD
                        and not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target(
                            request.user, cset.target
                        )
                    ):
                        # TODO: this will break if wrong user accesses
                        # it, but with the given flow, this is very
                        # unlikely to happen
                        logger.debug('User not recognised')
                        return JsonResponse(
                            {
                                'error_message': "You are not a member of the Target's proposal"
                            },
                        )

                    response_data['results']['cset_download_url'] = (
                        '/viewer/compound_set/%s' % cset.id
                    )
                    response_data['results']['pset_download_url'] = (
                        '/viewer/protein_set/%s' % cset.id
                    )

                    process_messages = results[2]
                    # set pandas options to display all column data
                    if len(next(iter(process_messages.values()))) > 0:
                        pd.set_option('display.max_colwidth', -1)
                        table = pd.DataFrame.from_dict(process_messages)
                        html_table = '''<p> Your data was uploaded with warnings</p>'''
                        html_table += table.to_html()
                        response_data['html'] = html_table

                    return JsonResponse(response_data)

                else:
                    # Upload/Update output tasks send back a tuple
                    response_data['validated'] = 'Validated'
                    cset_name = results[1]
                    cset = models.ComputedSet.objects.get(name=cset_name)

                    if (
                        settings.AUTHENTICATE_UPLOAD
                        and not _ISPYB_SAFE_QUERY_SET.user_is_member_of_target(
                            request.user, cset.target
                        )
                    ):
                        return Response(
                            {'error': "You are not a member of the Target's proposal"},
                            status=status.HTTP_403_FORBIDDEN,
                        )

                    response_data['results'] = {}
                    response_data['results']['cset_download_url'] = (
                        '/viewer/compound_set/%s' % cset.id
                    )
                    response_data['results']['pset_download_url'] = (
                        '/viewer/protein_set/%s' % cset.id
                    )

                    return JsonResponse(response_data)

            else:
                # Error output
                html_table = '''<p> Your data was <b>not</b> processed.</p>'''
                response_data['processed'] = 'None'
                response_data['html'] = html_table
                return JsonResponse(response_data)

        return JsonResponse(response_data)


# Start of ActionType
class ActionTypeView(viewsets.ModelViewSet):
    """View to retrieve information about action types available to users (GET).
    (api/action-types).
    """

    queryset = models.ActionType.objects.filter()
    serializer_class = serializers.ActionTypeSerializer

    # POST method allowed for flexibility in the PoC. In the final design we may want to prevent the POST/PUT methods
    # from being used
    # for action types so that these can only be updated via the admin panel.
    #    http_method_names = ['get', 'head']

    filterset_fields = '__all__'


# Start of Session Project
class SessionProjectView(
    mixins.UpdateModelMixin,
    mixins.CreateModelMixin,
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    """View to retrieve information about user projects (collection of sessions) (GET).
    Also used for saving project information (PUT, POST, PATCH).
    (api/session-projects).
    """

    queryset = models.SessionProject.objects.filter()
    filter_permissions = "target__project"
    filterset_fields = '__all__'

    def get_serializer_class(self):
        if self.request.method in ['GET']:
            return serializers.SessionProjectReadSerializer
        return serializers.SessionProjectWriteSerializer


class SessionActionsView(
    mixins.UpdateModelMixin,
    mixins.CreateModelMixin,
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    """View to retrieve information about actions relating to sessions_project (GET).
    Also used for saving project action information (PUT, POST, PATCH).
    (api/session-actions).
    """

    queryset = models.SessionActions.filter_manager.filter_qs()
    filter_permissions = "session_project__target__project"
    serializer_class = serializers.SessionActionsSerializer

    #   Note: jsonField for Actions will need specific queries - can introduce if needed.
    filterset_fields = ('id', 'author', 'session_project', 'last_update_date')


class SnapshotView(
    mixins.UpdateModelMixin,
    mixins.CreateModelMixin,
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    """View to retrieve information about user sessions (snapshots) (GET).
    Also used for saving session information (PUT, POST, PATCH). (api/snapshots)
    """

    queryset = models.Snapshot.filter_manager.filter_qs()
    filter_permissions = "session_project__target__project"
    filterset_class = filters.SnapshotFilter

    def get_serializer_class(self):
        if self.request.method in ['GET']:
            return serializers.SnapshotReadSerializer
        return serializers.SnapshotWriteSerializer


class SnapshotActionsView(
    mixins.UpdateModelMixin,
    mixins.CreateModelMixin,
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    """View to retrieve information about actions relating to snapshots (GET).
    Also used for saving snapshot action information (PUT, POST, PATCH).
    (api/snapshot-actions).
    """

    queryset = models.SnapshotActions.filter_manager.filter_qs()
    filter_permissions = "snapshot__session_project__target__project"
    serializer_class = serializers.SnapshotActionsSerializer

    #   Note: jsonField for Actions will need specific queries - can introduce if needed.
    filterset_fields = (
        'id',
        'author',
        'session_project',
        'snapshot',
        'last_update_date',
    )


class DesignSetCSVParser(BaseParser):
    """
    CSV parser class specific to design set csv spec -
    sets media_type for DSetUploadView to text/csv
    """

    media_type = 'text/csv'


class DesignSetUploadView(APIView):
    """Upload a design set (PUT) from a csv file"""

    parser_class = (DesignSetCSVParser,)

    def put(self, request, format=None):  # pylint: disable=redefined-builtin
        """Method to handle PUT request and upload a design set"""
        # Don't need...
        del format, request

        # Unsupported for now, as part of 1247 (securing endpoints)
        return HttpResponse(status=status.HTTP_404_NOT_FOUND)

        # BEGIN removed as part of 1247 (securing endpoints)
        #       This code is unused by the f/e

        # f = request.FILES['file']
        # set_type = request.PUT['type']
        # set_description = request.PUT['description']

        # # save uploaded file to temporary storage
        # name = f.name
        # path = default_storage.save('tmp/' + name, ContentFile(f.read()))
        # tmp_file = str(os.path.join(settings.MEDIA_ROOT, path))

        # df = pd.read_csv(tmp_file)
        # mandatory_cols = ['set_name', 'smiles', 'identifier', 'inspirations']
        # actual_cols = df.columns
        # for col in mandatory_cols:
        #     if col not in actual_cols:
        #         raise ParseError(
        #             "The 4 following columns are mandatory: set_name, smiles, identifier, inspirations"
        #         )

        # set_names, compounds = process_design_sets(df, set_type, set_description)

        # string = 'Design set(s) successfully created: '

        # length = len(set_names)
        # string += str(length) + '; '
        # for i in range(0, length):
        #     string += (
        #         str(i + 1)
        #         + ' - '
        #         + set_names[i]
        #         + ') number of compounds = '
        #         + str(len(compounds[i]))
        #         + '; '
        #     )

        # return HttpResponse(json.dumps(string))

        # END removed as part of 1247 (securing endpoints)


class ComputedSetView(
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    """Retrieve information about and delete computed sets."""

    queryset = models.ComputedSet.filter_manager.filter_qs()
    serializer_class = serializers.ComputedSetSerializer
    filter_permissions = "target__project"
    permission_classes = [IsObjectProposalMember]
    # permission_classes = [permissions.IsAuthenticated, IsObjectProposalMember]
    filterset_class = filters.ComputedSetFilter

    http_method_names = ['get', 'head', 'delete']

    def destroy(self, request, pk=None):
        """User provides the name of the ComputedSet (that's its primary key).
        We simply look it up and delete it, returning a standard 204 on success.
        """
        # Unused arguments
        del request

        computed_set = get_object_or_404(models.ComputedSet, pk=pk)
        computed_set.delete()
        return HttpResponse(status=204)

    @action(detail=True, methods=['get'])
    def download(self, request, pk=None):
        """Download a specific ComputedSet by ID."""

        try:
            computed_set = models.ComputedSet.objects.get(id=pk)
        except models.ComputedSet.DoesNotExist:
            return Response(
                # {'error': f"ComputedSet '{computed_set_id}' not found"},
                {'error': f"ComputedSet '{pk}' not found"},
                status=status.HTTP_404_NOT_FOUND,
            )

        if (
            computed_set.target.project.title
            not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(
                request.user, restrict_public_to_membership=False
            )
        ):
            return Response(
                {'error': "You have no access to the Project"},
                status=status.HTTP_403_FORBIDDEN,
            )

        zip_buffer = BytesIO()

        sdfs = models.ComputedSet.history.filter(
            written_sdf_filename__isnull=False,
        )
        pdbs = computed_set.computed_molecules.filter(pdb__isnull=True)

        # so now, get the file, and get the pdbs
        with zipfile.ZipFile(zip_buffer, 'a', zipfile.ZIP_DEFLATED) as ziparchive:
            for sdf in sdfs:
                sdf_file = Path(sdf.written_sdf_filename)
                if sdf_file.exists():
                    with open(sdf_file, 'rb') as contents:
                        ziparchive.writestr(str(sdf.submitted_sdf), contents.read())
                else:
                    ziparchive.writestr(f'{str(sdf.submitted_sdf)}_MISSING', r'')

            for f in pdbs:
                fpath = Path(settings.MEDIA_ROOT).joinpath(f.pdb_info.name)
                if fpath.is_file():
                    with open(fpath, 'rb') as contents:
                        ziparchive.writestr(f.get_filename(), contents.read())
                else:
                    ziparchive.writestr(f'{f.get_filename()}_MISSING', r'')

        response = HttpResponse(zip_buffer.getvalue(), content_type='application/zip')
        response['Content-Disposition'] = (
            'attachment; filename="%s"' % f'{computed_set.name}.zip'
        )
        response['Content-Length'] = zip_buffer.getbuffer().nbytes
        return response


class ComputedMoleculesView(ISPyBSafeQuerySet):
    """Retrieve information about computed molecules - 3D info (api/compound-molecules)."""

    queryset = models.ComputedMolecule.objects.all()
    serializer_class = serializers.ComputedMoleculeSerializer
    filter_permissions = "compound__project_id"
    filterset_fields = ('computed_set',)


class NumericalScoreValuesView(ISPyBSafeQuerySet):
    """View to retrieve information about numerical computed molecule scores
    (api/numerical-scores).
    """

    queryset = models.NumericalScoreValues.objects.all()
    serializer_class = serializers.NumericalScoreSerializer
    filter_permissions = "compound__compound__project_id"
    filterset_fields = ('compound', 'score')


class TextScoresView(ISPyBSafeQuerySet):
    """View to retrieve information about text computed molecule scores (api/text-scores)."""

    queryset = models.TextScoreValues.objects.all()
    serializer_class = serializers.TextScoreSerializer
    filter_permissions = "compound__compound__project_id"
    filterset_fields = ('compound', 'score')


class CompoundScoresView(ISPyBSafeQuerySet):
    """View to retrieve descriptions of scores for a given name or computed set."""

    queryset = models.ScoreDescription.objects.all()
    serializer_class = serializers.ScoreDescriptionSerializer
    filter_permissions = "computed_set__target__project"
    filterset_fields = ('computed_set', 'name')


class ComputedMolAndScoreView(ISPyBSafeQuerySet):
    """View to retrieve all information about molecules from a computed set
    along with all of their scores.
    """

    queryset = models.ComputedMolecule.objects.all()
    serializer_class = serializers.ComputedMolAndScoreSerializer
    filter_permissions = "compound__project_id"
    filterset_fields = ('computed_set',)


class DiscoursePostView(viewsets.ViewSet):
    """View to get and post to the Discourse platform

    example of input (GET) on local:

        http://127.0.0.1:8080/api/discourse_post/?post_title=Mpro%20First%20Project

    examples of input (POST raw data):

        {"category_name": "NewCategory", "parent_category_name": "Fragalysis targets", "category_colour": "0088CC",
        "category_text_colour": "FFFFFF", "post_title": "", "post_content": "", "post_tags":""}

        {"category_name": "NewCategory", "parent_category_name": "Fragalysis targets", "category_colour": "0088CC",
        "category_text_colour": "FFFFFF", "post_title": "New Topic Title 1",
        "post_content": "This is the first post that creates the topic - must be greater than 20 chars",
        "post_tags" :"[\\"tag1\\",\\"tag2\\"]"}

        {"category_name": "NewCategory", "parent_category_name": "Fragalysis targets", "category_colour": "0088CC",
        "category_text_colour": "FFFFFF", "post_title": "New Topic Title 1",
        "post_content": "This is a second post to New Topic Title 1", "post_tags" :"[]"}

    """

    serializer_class = serializers.DiscoursePostWriteSerializer

    def create(self, request):
        """Method to handle POST request and call discourse to create the post"""
        logger.info('+ DiscoursePostView.post')
        if not request.user.is_authenticated:
            content: Dict[str, Any] = {
                'error': 'Only authenticated users can post content to Discourse'
            }
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        data = request.data

        logger.info('+ DiscoursePostView.post %s', json.dumps(data))
        if data['category_name'] == '':
            category_details = None
        else:
            category_details = {
                'category_name': data['category_name'],
                'parent_name': data['parent_category_name'],
                'category_colour': data['category_colour'],
                'category_text_colour': data['category_text_colour'],
            }

        if data['post_title'] == '':
            post_details = None
        else:
            post_details = {
                'title': data['post_title'],
                'content': data['post_content'],
                'tags': json.loads(data['post_tags']),
            }

        error, post_url, error_message = create_discourse_post(
            request.user, category_details, post_details
        )

        logger.info('- DiscoursePostView.post')
        if error:
            return Response({"message": error_message})
        else:
            return Response({"Post url": post_url})

    def list(self, request):
        """Method to handle GET request and call discourse to list posts for a topic"""
        logger.info('+ DiscoursePostView.get')
        query_params = request.query_params
        logger.info('+ DiscoursePostView.get %s', json.dumps(query_params))

        discourse_api_key = settings.DISCOURSE_API_KEY

        if discourse_api_key:
            post_title = request.query_params.get('post_title', None)
            error, posts = list_discourse_posts_for_topic(post_title)
        else:
            logger.info('- DiscoursePostView.no key')
            return Response(
                {"message": "Discourse Not Available - No API key supplied"}
            )

        logger.info('- DiscoursePostView.get')
        if error:
            return Response({"message": "No Posts Found"})
        else:
            return Response({"Posts": posts})


class DictToCSVView(viewsets.ViewSet):
    """Takes a dictionary and returns a download link to a CSV file with the data."""

    serializer_class = serializers.DictToCsvSerializer

    def list(self, request):
        """Method to handle GET request.
        If the file exists it is returned and then removed."""
        file_url = request.GET.get('file_url', '')
        logger.info('DictToCsv file_url="%s"', file_url)

        # The file is expected to include a full path
        # to a file in the dicttocsv directory.
        real_file_url = os.path.realpath(file_url)
        if (
            os.path.commonpath([CSV_TO_DICT_DOWNLOAD_ROOT, real_file_url])
            != CSV_TO_DICT_DOWNLOAD_ROOT
        ):
            logger.warning(
                'DictToCsv path is invalid (file_url="%s" real_file_url="%s")',
                file_url,
                real_file_url,
            )
            return Response("Please provide a file_url for an existing DictToCsv file")
        elif not os.path.isfile(real_file_url):
            logger.warning(
                'DictToCsv file does not exist (file_url="%s" real_file_url="%s")',
                file_url,
                real_file_url,
            )
            return Response("The given DictToCsv file does not exist")

        with open(real_file_url, encoding='utf8') as csvfile:
            # return file and tidy up.
            response = HttpResponse(csvfile, content_type='text/csv')
            # response['Content-Disposition'] = 'attachment; filename=download.csv'
            filename = str(Path(real_file_url).name)
            response['Content-Disposition'] = f'attachment; filename={filename}'
            shutil.rmtree(os.path.dirname(real_file_url), ignore_errors=True)
            return response

    def create(self, request):
        """Method to handle POST request. Creates a file that the user
        is then expected to GET."""
        input_dict = request.data['dict']
        input_title = request.data['title']
        filename = request.data.get('filename', 'download.csv')

        logger.info('title="%s" input_dict size=%s', input_title, len(input_dict))
        if not input_dict:
            return Response({"message": "Please provide a dictionary"})
        else:
            file_url = create_csv_from_dict(input_dict, input_title, filename=filename)
        logger.info(
            'Created file_url="%s" (size=%s)',
            file_url,
            os.path.getsize(file_url),
        )

        return Response({"file_url": file_url})


# Classes Relating to Tags
class TagCategoryView(viewsets.ReadOnlyModelViewSet):
    """Set up and retrieve information about tag categories (api/tag_category)."""

    queryset = models.TagCategory.objects.all()
    serializer_class = serializers.TagCategorySerializer
    filterset_fields = ('id', 'category')


class SiteObservationTagView(
    mixins.UpdateModelMixin,
    mixins.CreateModelMixin,
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    """Set up/retrieve information about tags relating to Molecules (api/molecule_tag)"""

    queryset = models.SiteObservationTag.objects.all()
    filter_permissions = "target__project"
    serializer_class = serializers.SiteObservationTagSerializer
    filterset_fields = (
        'id',
        'tag',
        'category',
        'target',
        'site_observations',
        'mol_group',
    )


class PoseView(
    mixins.UpdateModelMixin,
    mixins.CreateModelMixin,
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    """Set up/retrieve information about Poses (api/poses)"""

    queryset = models.Pose.filter_manager.filter_qs()
    filter_permissions = "compound__project_id"
    serializer_class = serializers.PoseSerializer
    filterset_class = filters.PoseFilter


class SessionProjectTagView(
    mixins.UpdateModelMixin,
    mixins.CreateModelMixin,
    mixins.DestroyModelMixin,
    ISPyBSafeQuerySet,
):
    """Set up/retrieve information about tags relating to Session Projects."""

    queryset = models.SessionProjectTag.objects.all()
    filter_permissions = "target__project"
    serializer_class = serializers.SessionProjectTagSerializer
    filterset_fields = ('id', 'tag', 'category', 'target', 'session_projects')


class DownloadStructuresView(
    mixins.CreateModelMixin,
    ISPyBSafeQuerySet,
):
    """Uses a selected subset of the target data
    (proteins and booleans with suggested files) and creates a Zip file
    with the contents.

    Note that old zip files are removed after one hour.
    """

    queryset = models.Target.objects.all()
    serializer_class = serializers.DownloadStructuresSerializer
    filter_permissions = "project"
    filterset_fields = ('title', 'id')

    def list(self, request):
        """Method to handle GET request"""
        file_url = request.GET.get('file_url')

        if file_url:
            if models.DownloadLinks.objects.filter(file_url=file_url).first():
                logger.info('Found %s', file_url)
                assert os.path.isfile(file_url)

                file_name = os.path.basename(file_url)
                wrapper = FileWrapper(open(file_url, 'rb'))
                response = FileResponse(wrapper, content_type='application/zip')
                response['Content-Disposition'] = (
                    'attachment; filename="%s"' % file_name
                )
                response['Content-Length'] = os.path.getsize(file_url)
                return response
            else:
                content = {'message': 'file_url is not found'}
                return Response(content, status=status.HTTP_404_NOT_FOUND)

        content = {'message': 'Please provide file_url parameter from post response'}
        return Response(content, status=status.HTTP_404_NOT_FOUND)

    def create(self, request):
        """Method to handle POST requests that are used to initiate a target download.

        The user is permitted to download Targets they have access to (whether
        authenticated or not), and this is handled by the queryset logic later in
        this method.
        """
        logger.info('+ DownloadStructures.post')
        logger.debug('DownloadStructures.post.data: %s', request.data)

        erase_out_of_date_download_records()

        # Static files (i.e. links not removed)
        if request.data['file_url']:
            file_url = request.data['file_url']
            logger.info('Given file_url "%s"', file_url)
            existing_link = models.DownloadLinks.objects.filter(
                file_url=file_url
            ).first()

            if existing_link and existing_link.static_link:
                # A record exists, the file _must_ exist.
                file_url = existing_link.file_url
                logger.info('Existing static link found for file_url "%s"', file_url)
                assert os.path.isfile(file_url)
                return Response(
                    {"file_url": file_url},
                    status=status.HTTP_200_OK,
                )

            msg = 'file_url should only be provided for static files'
            logger.warning(msg)
            content = {'message': msg}
            return Response(content, status=status.HTTP_400_BAD_REQUEST)

        # Dynamic files
        if 'target_access_string' not in request.data:
            content = {
                'message': 'If no file_url, a target_access_string must be provided'
            }
            return Response(content, status=status.HTTP_400_BAD_REQUEST)

        if 'target_name' not in request.data:
            content = {
                'message': 'If no file_url, a target_name (title) must be provided'
            }
            return Response(content, status=status.HTTP_400_BAD_REQUEST)

        target_name = request.data['target_name']
        project_name = request.data['target_access_string']
        target = None
        logger.info('Given target_name "%s"', target_name)

        # Check target_name is valid:
        try:
            project = models.Project.objects.get(title=project_name)
        except models.Project.DoesNotExist:
            msg = f'Project "{project_name}" does not exist'
            logger.warning(msg)
            content = {'message': msg}
            return Response(content, status=status.HTTP_404_NOT_FOUND)

        if project.title not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(
            request.user
        ):
            msg = f'User "{request.user.username}" is not a member of {project_name}'
            logger.warning(msg)
            content = {'message': msg}
            return Response(content, status=status.HTTP_404_NOT_FOUND)

        try:
            target = self.queryset.get(title=target_name, project=project)
        except models.Target.DoesNotExist:
            msg = f'Target "{target_name}" does not exist under project {project.title}'
            logger.warning(msg)
            content = {'message': msg}
            return Response(content, status=status.HTTP_404_NOT_FOUND)

        logger.info('Found Target record %r', target)

        proteins_list = [
            p.strip() for p in request.data.get('proteins', '').split(',') if p
        ]
        if proteins_list:
            logger.info('Given %s Proteins %s', len(proteins_list), proteins_list)
            logger.info('Looking for SiteObservation records for given Proteins...')

            site_obvs = models.SiteObservation.objects.filter(
                experiment__experiment_upload__target=target,
                code__in=proteins_list,
            )

            missing_obvs = set(proteins_list).difference(
                set(site_obvs.values_list('code', flat=True))
            )
            if missing_obvs:
                logger.warning(
                    'Could not find SiteObservation record for "%s"',
                    missing_obvs,
                )

        else:
            logger.info('Request had no Proteins')
            logger.info('Looking for Protein records for %r...', target)
            site_obvs = models.SiteObservation.objects.filter(
                experiment__experiment_upload__target=target
            )

        if not site_obvs.exists():
            content = {
                'message': 'Please enter list of valid protein codes '
                'for target: {}, proteins: {}'.format(target.title, proteins_list)
            }
            return Response(content, status=status.HTTP_404_NOT_FOUND)

        # Forced errors?
        if have_infection(INFECTION_STRUCTURE_DOWNLOAD):
            content = {'message': f'Download Error! ({INFECTION_STRUCTURE_DOWNLOAD})'}
            return Response(content, status=status.HTTP_404_NOT_FOUND)

        filename_url = create_or_return_download_link(request, target, site_obvs)
        assert filename_url is not None
        return Response({"file_url": filename_url})


class UploadExperimentUploadView(viewsets.ViewSet):
    http_method_names = ('post',)
    serializer_class = serializers.TargetExperimentWriteSerializer

    def get_view_name(self):
        return "Upload Target Experiments"

    def create(self, request, *args, **kwargs):
        logger.info("+ UploadTargetExperiments.create called")
        logger.debug('request.data :%s', request.data)

        # logger.debug('request.POST :%s', request.POST)
        logger.debug('request.user :%s', request.user)

        del args, kwargs

        serializer = self.serializer_class(data=request.data)
        if not serializer.is_valid():
            logger.debug("serializer not valid: %s", serializer.errors)
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        logger.debug("Serializer validated_data=%s", serializer.validated_data)
        logger.debug("User=%s", self.request.user)

        target_access_string = serializer.validated_data['target_access_string']

        if settings.AUTHENTICATE_UPLOAD:
            if self.request.user.username == 'asap-service':
                logger.warning(
                    'Upload attempted with "%s" service account, trying uploader-supplied user',
                    self.request.user.username,
                )
                if 'django-user' in request.headers.keys():
                    try:
                        user = get_user_model().objects.get(
                            username=request.headers['django-user']
                        )
                    except get_user_model().DoesNotExist:
                        msg = (
                            f'Upload from "{self.request.user.username}" '
                            + 'service account but fragalysis user not found'
                        )
                        logger.error(msg)
                        return Response(
                            {'error': msg}, status=status.HTTP_403_FORBIDDEN
                        )
                else:
                    msg = (
                        f'Upload from "{self.request.user.username}" service '
                        'account but fragalysis user not supplied'
                    )
                    logger.error(msg)
                    return Response({'error': msg}, status=status.HTTP_403_FORBIDDEN)

            else:
                user = self.request.user

            if not user.is_authenticated:
                return redirect(settings.LOGIN_URL)
            else:
                if (
                    target_access_string
                    not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(
                        user, restrict_public_to_membership=True
                    )
                ):
                    return Response(
                        {
                            "target_access_string": [
                                f"You are not authorized to upload data to '{target_access_string}'"
                            ]
                        },
                        status=status.HTTP_403_FORBIDDEN,
                    )

        if 'data_version' in serializer.validated_data.keys():
            try:
                major, minor = split_version(serializer.validated_data['data_version'])
            except ValueError as exc:
                return Response(
                    {
                        'success': False,
                        'message': exc.args[0],
                    },
                    status=status.HTTP_200_OK,
                )
            try:
                target_name = serializer.validated_data['target_name']
            except KeyError:
                return Response(
                    {'success': False, 'message': 'Target name not given'},
                    status=status.HTTP_200_OK,
                )

            val_result, msg = validate_data_version(
                major, minor, target_name=target_name, project_name=target_access_string
            )
            return Response(
                {
                    'success': val_result,
                    'message': msg,
                },
                status=status.HTTP_200_OK,
            )

        filename = serializer.validated_data['file']

        # memo to self: cannot use TemporaryDirectory here because task
        temp_path = Path(settings.MEDIA_ROOT).joinpath('tmp')
        temp_path.mkdir(exist_ok=True)
        target_file = temp_path.joinpath(filename.name)
        handle_uploaded_file(target_file, filename)

        celery_app = Celery("fragalysis")
        celery_app.config_from_object("django.conf:settings", namespace="CELERY")
        inspect = celery_app.control.inspect()
        ping = inspect.ping()

        if not ping:
            # celery not active in local development. log a warning
            # and try to run the task anyway
            logger.warning('Celery not running!')

        task = task_load_target.delay(
            data_bundle=str(target_file),
            proposal_ref=target_access_string,
            user_id=request.user.pk,
        )
        logger.info("+ UploadTargetExperiments.create got Celery id %s", task.task_id)

        url = reverse('viewer:task_status', kwargs={'task_id': task.task_id})
        # as it launches task, I think 202 is more appropriate
        return Response({'task_status_url': url}, status=status.HTTP_202_ACCEPTED)


class TaskStatusView(APIView):
    def get(self, request, task_id, *args, **kwargs):
        """Given a task_id (a string) we try to return the status of the task,
        trying to handle unknown tasks as best we can.
        """
        # Unused arguments
        del args, kwargs

        logger.debug("task_id=%s", task_id)

        if not request.user.is_authenticated and settings.AUTHENTICATE_UPLOAD:
            content: Dict[str, Any] = {
                'error': 'Only authenticated users can check the task status'
            }
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # task_id is a UUID, but Celery expects a string
        task_id_str = str(task_id)
        result = None
        try:
            result = AsyncResult(task_id_str)
        except TimeoutError:
            error = {'error': 'Task query timed out. Try again later'}
            return Response(error, status=status.HTTP_408_REQUEST_TIMEOUT)
        # Handle failure cases
        if not result:
            error = {'error': 'Task query returned nothing, contact your administrator'}
            return Response(error, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

        # Extract messages from result's info attribute
        # (if it has an info attribute)
        messages = []
        if hasattr(result, 'info'):
            if isinstance(result.info, dict):
                # check if user is allowed to view task info
                proposal = result.info.get('proposal_ref', '')

                if proposal not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(
                    request.user
                ):
                    return Response(
                        {'error': 'You are not a member of the proposal f"proposal"'},
                        status=status.HTTP_403_FORBIDDEN,
                    )

                messages = result.info.get('description', [])
            elif isinstance(result.info, list):
                # this path should never materialize
                logger.error('result.info attribute list instead of dict')
                return Response(
                    {'error': 'You are not a member of the proposal f"proposal"'},
                    status=status.HTTP_403_FORBIDDEN,
                )
                # messages = result.info

        started = result.state != 'PENDING'
        finished = result.ready()
        task_status = "UNKNOWN"
        if finished and messages:
            # The task is considered to have failed if the word 'FAILED'
            # is in the last line of the message (regardless of case)
            task_status = "FAILED" if 'failed' in messages[-1].lower() else "SUCCESS"
        elif started:
            task_status = "RUNNING"

        data = {
            'started': started,
            'finished': finished,
            'status': task_status,
            'messages': messages,
        }
        return JsonResponse(data)


class DownloadExperimentUploadView(viewsets.ModelViewSet):
    serializer_class = serializers.TargetExperimentDownloadSerializer
    permission_class = [permissions.IsAuthenticated]
    http_method_names = ('post',)

    def get_view_name(self):
        return "Download Target Experiments"

    def create(self, request, *args, **kwargs):
        # Unused arguments
        del args, kwargs

        logger.info("+ DownloadExperimentUploadView.create called")

        serializer = self.get_serializer_class()(data=request.data)
        if serializer.is_valid():
            # To permit a download the user must be a member of the target's proposal
            # (or the proposal must be open)
            project: models.Project = serializer.validated_data['project']
            if project.title not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(
                request.user, restrict_public_to_membership=False
            ):
                return Response(
                    {'error': "You have no access to the Project"},
                    status=status.HTTP_403_FORBIDDEN,
                )
            target: models.Target = serializer.validated_data['target']
            if project != target.project.title:
                return Response(
                    {'error': "The Target is not part of the given Project"},
                    status=status.HTTP_403_FORBIDDEN,
                )

            # Now we have to search for an ExperimentUpload that matches the Target
            # and the filename combination.
            filename = serializer.validated_data['filename']
            exp_uploads: List[
                models.ExperimentUpload
            ] = models.ExperimentUpload.objects.filter(
                target=target,
                file=filename,
            )
            if len(exp_uploads) > 1:
                return Response(
                    {
                        'error': "More than one ExperimentUpload matches your Target and Filename"
                    },
                    status=status.HTTP_400_INTERNAL_SERVER_ERROR,
                )
            elif len(exp_uploads) == 0:
                return Response(
                    {'error': "No ExperimentUpload matches your Target and Filename"},
                    status=status.HTTP_404_NOT_FOUND,
                )
            # Use the only experiment upload found.
            # We don't need the user's filename any more,
            # it's embedded in the ExperimentUpload's file field.
            exp_upload: models.ExperimentUpload = exp_uploads[0]
            file_path = exp_upload.get_download_path()
            logger.info(
                "Found exp_upload=%s file_path=%s",
                exp_upload,
                file_path,
            )
            if not file_path.exists():
                return Response(
                    {'error': f"TargetExperiment file '{filename}' not found"},
                    status=status.HTTP_404_NOT_FOUND,
                )
            # Finally, return the file
            wrapper = FileWrapper(open(file_path, 'rb'))
            response = FileResponse(wrapper, content_type='application/zip')
            response['Content-Disposition'] = (
                'attachment; filename="%s"' % file_path.name
            )
            response['Content-Length'] = os.path.getsize(file_path)
            return response

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


class ExperimentUploadView(ISPyBSafeQuerySet):
    queryset = models.ExperimentUpload.upload_manager.annotated_qs()
    serializer_class = serializers.TargetExperimentReadSerializer
    permission_class = [permissions.IsAuthenticated]
    filterset_class = filters.ExperimentUploadFilter
    filter_permissions = "target__project"
    http_method_names = ('get',)


class SiteObservationView(ISPyBSafeQuerySet):
    queryset = models.SiteObservation.filter_manager.filter_qs().filter(
        superseded=False
    )
    serializer_class = serializers.SiteObservationReadSerializer
    filterset_class = filters.SiteObservationFilter
    filter_permissions = "experiment__experiment_upload__project"


class CanonSiteView(ISPyBSafeQuerySet):
    queryset = models.CanonSite.filter_manager.filter_qs().filter(superseded=False)
    serializer_class = serializers.CanonSiteReadSerializer
    filterset_class = filters.CanonSiteFilter
    filter_permissions = (
        "ref_conf_site__ref_site_observation__experiment__experiment_upload__project"
    )


class ExperimentView(ISPyBSafeQuerySet):
    queryset = models.Experiment.filter_manager.filter_qs()
    serializer_class = serializers.ExperimentReadSerializer
    filterset_class = filters.ExperimentFilter
    filter_permissions = "experiment_upload__project"


class CanonSiteConfView(ISPyBSafeQuerySet):
    queryset = models.CanonSiteConf.filter_manager.filter_qs().filter(superseded=False)
    serializer_class = serializers.CanonSiteConfReadSerializer
    filterset_class = filters.CanonSiteConfFilter
    filter_permissions = "ref_site_observation__experiment__experiment_upload__project"


class XtalformSiteView(ISPyBSafeQuerySet):
    queryset = models.XtalformSite.filter_manager.filter_qs().filter(superseded=False)
    serializer_class = serializers.XtalformSiteReadSerializer
    filterset_class = filters.XtalformSiteFilter
    filter_permissions = "canon_site__ref_conf_site__ref_site_observation__experiment__experiment_upload__project"


class JobFileTransferView(viewsets.ModelViewSet):
    """Squonk Job file transfer (api/job_file_transfer)"""

    queryset = models.JobFileTransfer.objects.filter()
    filter_permissions = "target__project"
    filterset_fields = (
        'id',
        'snapshot',
        'target',
        'user',
        'squonk_project',
        'transfer_status',
    )

    def get_serializer_class(self):
        if self.request.method in ['GET']:
            return serializers.JobFileTransferReadSerializer
        # (POST, PUT, PATCH)
        return serializers.JobFileTransferWriteSerializer

    def create(self, request):
        """Method to handle POST request"""
        logger.info('+ JobFileTransferView.post')
        # Only authenticated users can transfer files to squonk
        user = self.request.user
        if not user.is_authenticated:
            content: Dict[str, Any] = {
                'error': 'Only authenticated users can transfer files'
            }
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Can't use this method if the Squonk2 agent is not configured
        sq2a_rv = _SQ2A.configured()
        if not sq2a_rv.success:
            content = {'error': f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Collect expected API parameters....
        access_id = request.data['access']  # The access ID/legacy Project record title
        target_id = request.data['target']
        snapshot_id = request.data['snapshot']
        session_project_id = request.data['session_project']

        logger.info('+ user="%s" (id=%s)', user.username, user.id)
        logger.info('+ access_id=%s', access_id)
        logger.info('+ target_id=%s', target_id)
        logger.info('+ snapshot_id=%s', snapshot_id)
        logger.info('+ session_project_id=%s', session_project_id)

        # Check the user can use this Squonk2 facility.
        # To do this we need to setup a couple of API parameter objects.
        sq2a_common_params: CommonParams = CommonParams(
            user_id=user.id,
            access_id=access_id,
            session_id=session_project_id,
            target_id=target_id,
        )
        sq2a_send_params: SendParams = SendParams(
            common=sq2a_common_params, snapshot_id=snapshot_id
        )
        sq2a_rv = _SQ2A.can_send(sq2a_send_params)
        if not sq2a_rv.success:
            content = {'error': f'You cannot do this ({sq2a_rv.msg})'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Check the existence of the files that are expected to be transferred
        error, protein_files, compound_files = validate_file_transfer_files(request)
        if error:
            return Response(error['message'], status=error['status'])
        assert protein_files
        assert compound_files

        # Create new file transfer job
        logger.info('+ Calling ensure_project() to get the Squonk2 Project...')

        # This requires a Squonk2 Project (created by the Squonk2Agent).
        # It may be an existing project, or it might be a new project.
        common_params = CommonParams(
            user_id=user.id,
            access_id=access_id,
            target_id=target_id,
            session_id=session_project_id,
        )
        sq2_rv = _SQ2A.ensure_project(common_params)
        if not sq2_rv.success:
            msg = (
                f'Failed to get/create a Squonk2 Project'
                f' for User "{user.username}", Access ID {access_id},'
                f' Target ID {target_id}, and SessionProject ID {session_project_id}.'
                f' Got "{sq2_rv.msg}".'
                ' Cannot continue'
            )
            content = {'message': msg}
            logger.error(msg)
            return Response(content, status=status.HTTP_404_NOT_FOUND)

        # The Squonk2Project record is in the response msg
        squonk2_project_uuid = sq2_rv.msg.uuid
        squonk2_unit_name = sq2_rv.msg.unit.name
        squonk2_unit_uuid = sq2_rv.msg.unit.uuid
        logger.info(
            '+ ensure_project() returned Project uuid=%s (unit="%s" unit_uuid=%s)',
            squonk2_project_uuid,
            squonk2_unit_name,
            squonk2_unit_uuid,
        )

        job_transfer = models.JobFileTransfer()
        job_transfer.user = request.user
        job_transfer.proteins = [str(path_and_file) for path_and_file in protein_files]
        job_transfer.compounds = [
            str(path_and_file) for path_and_file in compound_files
        ]
        # We should use a foreign key,
        # but to avoid migration issues with the existing code
        # we continue to use the project UUID string field.
        job_transfer.squonk_project = squonk2_project_uuid
        job_transfer.target = models.Target.objects.get(id=target_id)
        job_transfer.snapshot = models.Snapshot.objects.get(id=snapshot_id)

        # The 'transfer target' (a sub-directory of the transfer root)
        # For example the root might be 'fragalysis-files'
        # and the target may be `CD44MMA` so the targets will be written to the
        # Squonk project at fragalysis-files/CD44MMA
        assert job_transfer.target
        assert job_transfer.target.title
        transfer_target = job_transfer.target.title
        logger.info('+ job_transfer.target=%s', transfer_target)

        job_transfer.transfer_status = 'PENDING'
        job_transfer.transfer_datetime = None
        job_transfer.transfer_progress = None
        job_transfer.save()

        # The root (in the Squonk project) where files will be written for this Job.
        # This will be something "${SQUONK2_MEDIA_DIRECTORY}/hjyx", where "hjyx" is
        # a randomly-generated 4-character sequence for the given transfer assigned
        # when the JobFileTransfer record is created.
        transfer_root = os.path.join(
            settings.SQUONK2_MEDIA_DIRECTORY, job_transfer.sub_path
        )
        logger.info(
            '+ Starting transfer (celery) (job_transfer.id=%s transfer_root=%s)...',
            job_transfer.id,
            transfer_root,
        )
        job_transfer_task = process_job_file_transfer.delay(
            request.session['oidc_access_token'], job_transfer.id
        )

        content = {
            'id': job_transfer.id,
            'transfer_root': transfer_root,
            'transfer_target': transfer_target,
            'transfer_status': job_transfer.transfer_status,
            'transfer_task_id': str(job_transfer_task),
        }
        return Response(content, status=status.HTTP_202_ACCEPTED)


class JobConfigView(viewsets.ReadOnlyModelViewSet):
    """Calls Squonk to get a requested job configuration"""

    def list(self, request):
        """Method to handle GET request"""
        query_params = request.query_params
        logger.info('+ JobConfigView.get: %s', json.dumps(query_params))

        # Only authenticated users can have squonk jobs
        user = self.request.user
        if not user.is_authenticated:
            content = {'Only authenticated users can access squonk jobs'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Can't use this method if the squonk variables are not set!
        sqa_rv = _SQ2A.configured()
        if not sqa_rv.success:
            content = {f'The Squonk2 Agent is not configured ({sqa_rv.msg})'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        job_collection = request.query_params.get('job_collection', None)
        job_name = request.query_params.get('job_name', None)
        job_version = request.query_params.get('job_version', None)

        content = get_squonk_job_config(
            request,
            job_collection=job_collection,
            job_name=job_name,
            job_version=job_version,
        )
        if not content:
            content = {
                f'No such job configuration (job_collection={job_collection}, job_name={job_name}, version={job_version})'
            }
            return Response(content, status=status.HTTP_404_NOT_FOUND)

        return Response(content)


class JobOverrideView(viewsets.ModelViewSet):
    queryset = models.JobOverride.objects.all().order_by('-id')

    def get_serializer_class(self):
        if self.request.method in ['GET']:
            return serializers.JobOverrideReadSerializer
        return serializers.JobOverrideWriteSerializer

    def create(self, request):
        logger.info('+ JobOverrideView.post')
        # Only authenticated users can transfer files to sqonk
        user = self.request.user
        if not user.is_authenticated:
            content: Dict[str, Any] = {
                'error': 'Only authenticated users can provide Job overrides'
            }
            return Response(content, status=status.HTTP_403_FORBIDDEN)
        if not user.is_staff:
            content = {'error': 'Only STAFF (Admin) users can provide Job overrides'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Override is expected to be a JSON string,
        # but protect against format issues
        override = request.data['override']
        try:
            override_dict = json.loads(override)
        except ValueError:
            content = {'error': 'The override is not valid JSON'}
            return Response(content, status=status.HTTP_400_BAD_REQUEST)
        # We could use a schema but that's comlex.
        # For now let's just insist on some key fields in the provided override: -
        # - global
        # - fragalysis-jobs
        if "global" not in override_dict:
            content = {'error': 'The override does not contain a "global" key'}
            return Response(content, status=status.HTTP_400_BAD_REQUEST)
        if "fragalysis-jobs" not in override_dict:
            content = {'error': 'The override does not contain a "fragalysis-jobs" key'}
            return Response(content, status=status.HTTP_400_BAD_REQUEST)
        if type(override_dict["fragalysis-jobs"]) != list:
            content = {'error': 'The override "fragalysis-jobs" key is not a list'}
            return Response(content, status=status.HTTP_400_BAD_REQUEST)

        job_override = models.JobOverride()
        job_override.override = override_dict
        job_override.author = user
        job_override.save()

        return Response({"id": job_override.id}, status=status.HTTP_201_CREATED)


class JobRequestView(viewsets.ModelViewSet):
    queryset = models.JobRequest.objects.filter()

    def get_serializer_class(self):
        if self.request.method in ['GET']:
            return serializers.JobRequestReadSerializer
        # (POST, PUT, PATCH)
        return serializers.JobRequestWriteSerializer

    def list(self, request):
        logger.info('+ JobRequestView.list')

        user = self.request.user
        if not user.is_authenticated:
            content: Dict[str, Any] = {
                'error': 'Only authenticated users can access squonk jobs'
            }
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Can't use this method if the Squonk2 agent is not configured
        sq2a_rv = _SQ2A.configured()
        if not sq2a_rv.success:
            content = {'error': f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        results = []
        if snapshot_id := request.query_params.get('snapshot', None):
            snapshot_msg: str = f'snapshot_id={snapshot_id}'
            logger.info(snapshot_msg)
            job_requests = models.JobRequest.objects.filter(snapshot=int(snapshot_id))
        else:
            snapshot_msg = 'snapshot_id=(unset)'
            job_requests = models.JobRequest.objects.all()

        if not job_requests:
            logger.info('No JobRequests found (%s)', snapshot_msg)
        else:
            logger.info('Found %d JobRequests (%s)', len(job_requests), snapshot_msg)

        # Iterate through each record, for JobRequests that are not 'finished'
        # we call into Squonk to get an update. We then return the (possibly) updated
        # records to the caller. Depending on the configuration we may also handle
        # retrieval of Job results (normally handled by the JobRequest callback).

        for jr in job_requests:
            logger.info(
                'Processing JobRequest id=%d (project=%s snapshot_id=%s)',
                jr.id,
                jr.project.title,
                snapshot_id,
            )

            # Skip any JobRequests the user does not have access to
            if not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals(
                user, [jr.project.title]
            ):
                logger.info('id=%d (access not granted)', jr.id)
                continue

            # An opportunity to update JobRequest timestamps?
            # And handle any results (if configured to do so)
            if jr.job_finish_datetime:
                logger.info(
                    'id=%d has already finished (job_finish_datetime=%s)',
                    jr.id,
                    jr.job_finish_datetime,
                )
            else:
                logger.info(
                    'id=%s has not finished (job_status=%s)',
                    jr.id,
                    jr.job_status,
                )

                # The Job's not finished, an opportunity to call into Squonk
                # To get the current status. To do this we'll need
                # the 'callback context' supplied when launching the Job.
                logger.info(
                    'id=%s, code=%s getting update from Squonk...',
                    jr.id,
                    jr.code,
                )

                # We need to make the call to Squonk2 as the API user.
                user_auth_token = request.session['oidc_access_token']
                sq2a_rv = _SQ2A.get_instance_execution_status(user_auth_token, jr.code)
                # If the job's now finished, update the record.
                # If the call was successful we'll get None (not finished),
                # 'LOST', 'SUCCESS' or 'FAILURE'
                if not sq2a_rv.success:
                    logger.warning(
                        'id=%s, code=%s check failed (%s)',
                        jr.id,
                        jr.code,
                        sq2a_rv.msg,
                    )

                elif sq2a_rv.msg and sq2a_rv.msg == 'LOST':
                    logger.info(
                        'id=%s, code=%s job lost (%s) or currently unknown',
                        jr.id,
                        jr.code,
                        sq2a_rv.msg,
                    )

                elif sq2a_rv.msg:
                    # A change of STATUS - and it's stopped!
                    logger.info(
                        'id=%s code=%s new status is %s',
                        jr.id,
                        jr.code,
                        sq2a_rv.msg,
                    )

                    # Our best guess at the transition time (the time now).
                    # The actual Job transition time may have been earlier.
                    transition_time_utc = datetime.now(timezone.utc)
                    jr.job_status = sq2a_rv.msg
                    jr.job_status_datetime = transition_time_utc
                    if not jr.job_start_datetime:
                        jr.job_start_datetime = transition_time_utc
                    if not jr.job_finish_datetime:
                        jr.job_finish_datetime = transition_time_utc
                    jr.save()

                    if (
                        jr.job_status == 'SUCCESS'
                        and settings.SQUONK2_REFRESH_SHOULD_RETRIEVE_RESULTS
                    ):
                        logger.info("id=%s running job_success_handler()...")
                        _ = job_success_handler(jr, transition_time_utc)

                else:
                    logger.info("id=%s code=%s job is running", jr.id, jr.code)

            serializer = serializers.JobRequestReadSerializer(jr)
            results.append(serializer.data)

        num_results = len(results)
        logger.info('num_results=%s', num_results)

        # Simulate the original paged API response...
        content = {
            'count': num_results,
            'next': None,
            'previous': None,
            'results': results,
        }
        return Response(content, status=status.HTTP_200_OK)

    def create(self, request):
        logger.info('+ JobRequestView.post')
        # Only authenticated users can create squonk job requests
        # (unless 'AUTHENTICATE_UPLOAD' is False in settings.py)
        user = self.request.user
        if not user.is_authenticated and settings.AUTHENTICATE_UPLOAD:
            content: Dict[str, Any] = {'error': 'Only authenticated users can run jobs'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Can't use this method if the Squonk2 agent is not configured
        sq2a_rv = _SQ2A.configured()
        if not sq2a_rv.success:
            content = {'error': f'The Squonk2 Agent is not configured ({sq2a_rv.msg})'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Collect expected API parameters....
        target_id = request.data['target']
        snapshot_id = request.data['snapshot']
        session_project_id = request.data['session_project']
        access_id = request.data['access']  # The access ID/legacy Project record

        logger.info('+ user="%s" (id=%s)', user.username, user.id)
        logger.info('+ access_id=%s', access_id)
        logger.info('+ target_id=%s', target_id)
        logger.info('+ snapshot_id=%s', snapshot_id)
        logger.info('+ session_project_id=%s', session_project_id)

        # Project must exist.
        project: Optional[models.Project] = models.Project.objects.filter(
            id=access_id
        ).first()
        if not project:
            content = {'error': f'Access ID (Project) {access_id} does not exist'}
            return Response(content, status=status.HTTP_404_NOT_FOUND)
        # The user must be a member of the target access string.
        # (when AUTHENTICATE_UPLOAD is set)
        if (
            settings.AUTHENTICATE_UPLOAD
            and not _ISPYB_SAFE_QUERY_SET.user_is_member_of_any_given_proposals(
                user, [project.title]
            )
        ):
            content = {'error': f"You are not a member of '{project.title}'"}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Check the user can use this Squonk2 facility.
        # To do this we need to setup a couple of API parameter objects.
        # We don't (at this point) care about the Job spec or callback URL.
        sq2a_common_params: CommonParams = CommonParams(
            user_id=user.id,
            access_id=access_id,
            session_id=session_project_id,
            target_id=target_id,
        )
        sq2a_run_job_params: RunJobParams = RunJobParams(
            common=sq2a_common_params, job_spec=None, callback_url=None
        )
        sq2a_rv = _SQ2A.can_run_job(sq2a_run_job_params)
        if not sq2a_rv.success:
            content = {'error': f'You cannot do this ({sq2a_rv.msg})'}
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        try:
            job_id, squonk_url_ext = create_squonk_job(request)
        except ValueError as error:
            logger.info('Job Request failed: %s', error)
            content = {'error': str(error)}
            return Response(content, status=status.HTTP_400_BAD_REQUEST)

        logger.info('SUCCESS (job_id=%s squonk_url_ext=%s)', job_id, squonk_url_ext)

        content = {'id': job_id, 'squonk_url_ext': squonk_url_ext}
        return Response(content, status=status.HTTP_202_ACCEPTED)


class JobCallBackView(viewsets.ModelViewSet):
    """View to allow the Squonk system to update the status and job information for a
    specific job identified by a UUID.
    """

    queryset = models.JobRequest.objects.all()
    lookup_field = "code"
    http_method_names = ['get', 'head', 'put']

    def get_serializer_class(self):
        """Determine which serializer to use based on whether the request is a GET or a PUT

        Returns
        -------
        Serializer (rest_framework.serializers.ModelSerializer):
            - if GET: `viewer.serializers.JobCallBackWriteSerializer`
            - if other: `viewer.serializers.JobCallBackWriteSerializer`
        """
        if self.request.method in ['GET']:
            # GET
            return serializers.JobCallBackReadSerializer
        # PUT
        return serializers.JobCallBackWriteSerializer

    def update(self, request, code=None):
        """Response to a PUT on the Job-Callback.
        We're given a 'code' which we use to lookup the corresponding JobRequest
        (there'll only be one).
        """

        jr = models.JobRequest.objects.get(code=code)
        logger.info('+ JobCallBackView.update(code=%s) jr=%s', code, jr)

        if settings.SQUONK2_REFRESH_SHOULD_RETRIEVE_RESULTS:
            # Result handling is done in the JobRequest refresh.
            logger.warning(
                '- SQUONK2_REFRESH_SHOULD_RETRIEVE_RESULTS is set, ignoring callbacks'
            )
            return HttpResponse(status=204)

        # request.data is rendered as a dictionary
        if not request.data:
            logger.warning('- code=%s ignoring (no data)', code)
            return HttpResponse(status=204)

        j_status = request.data['job_status']
        # Get the appropriate SQUONK_STATUS...
        status_changed = False
        for squonk_status in models.JobRequest.SQUONK_STATUS:
            if squonk_status[0] == j_status and jr.job_status != j_status:
                jr.job_status = squonk_status[1]
                status_changed = True
                break

        if not status_changed:
            logger.info(
                '- code=%s status=%s ignoring (no status change)',
                code,
                status,
            )
            return HttpResponse(status=204)

        # This is now a chance to safely set the squonk_url_ext using the instance ID
        # present in the callback (if it's not already set). The instance is
        # placed at the end of the string, and is expected to be found
        # by process_compound_set_file() which loads the output file back
        # into Fragalysis
        if not jr.squonk_url_ext:
            jr.squonk_url_ext = create_squonk_job_request_url(
                request.data['instance_id']
            )
            logger.info(
                "code=%s jr.squonk_url_ext='%s'",
                code,
                jr.squonk_url_ext,
            )

        # Update the state transition time,
        # assuming UTC.
        transition_time = request.data.get('state_transition_time')
        if not transition_time:
            transition_time = str(datetime.now(timezone.utc))
            logger.warning(
                "code=%s callback is missing state_transition_time (using '%s')",
                code,
                transition_time,
            )
        transition_time_utc = parse(transition_time).replace(tzinfo=pytz.UTC)
        jr.job_status_datetime = transition_time_utc

        logger.info(
            'code=%s status=%s transition_time=%s (new status)',
            code,
            status,
            transition_time,
        )

        # If the Job's start-time is not set, set it.
        if not jr.job_start_datetime:
            logger.info(
                'code=%s setting job START datetime (%s)',
                code,
                transition_time,
            )
            jr.job_start_datetime = transition_time_utc

        # Set the Job's finish time (once) if it looks lie the Job's finished.
        # We can assume the Job's finished if the status is one of a number
        # of values...
        if not jr.job_finish_datetime and status in ('SUCCESS', 'FAILURE', 'REVOKED'):
            logger.info(
                'code=%s Setting job FINISH datetime (%s)',
                code,
                transition_time,
            )
            jr.job_finish_datetime = transition_time_utc

        # Save the JobRequest record before going further.
        jr.save()

        if j_status != 'SUCCESS':
            # Go no further unless SUCCESS
            logger.info('- code=%s waiting for SUCCESS', code)
            return HttpResponse(status=204)

        # SUCCESS ... automatic upload?
        return job_success_handler(jr, transition_time_utc)


class JobAccessView(viewsets.GenericViewSet, mixins.ListModelMixin):
    """JobAccess (api/job_access)

    Django view that calls Squonk to allow a user (who is able to see a Job)
    the ability to access that Job in Squonk. To be successful the user
    must have access to the corresponding Fragalysis Project. This can be called by
    the Job 'owner', who always has access.
    """

    def list(self, request):
        """Method to handle a general GET request. All we do here is custom logic,
        the user cannot use this endpoint to get anything, it simply provides
        user-access to a Job in Squonk."""
        query_params = request.query_params
        logger.info('+ JobAccessView/GET %s', json.dumps(query_params))

        err_response: Dict[str, Any] = {'accessible': False}
        ok_response = {'accessible': True, 'error': ''}

        # Only authenticated users can have squonk jobs
        user = self.request.user
        if not user.is_authenticated:
            content = {
                'accessible': False,
                'error': 'Only authenticated users can access Squonk Jobs',
            }
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        # Can't use this method if the squonk variables are not set!
        sqa_rv = _SQ2A.configured()
        if not sqa_rv.success:
            err_response['error'] = f'Squonk is not available ({sqa_rv.msg})'
            return Response(err_response, status=status.HTTP_403_FORBIDDEN)

        # Get the JobRequest Record form the supplied record ID
        jr_id_str = request.query_params.get('job_request_id', None)
        if not jr_id_str or not jr_id_str.isdigit():
            err_response['error'] = f'The JobRequest ID ({jr_id_str}) is not valid'
            return Response(err_response, status=status.HTTP_400_BAD_REQUEST)
        jr_id = int(jr_id_str)
        if jr_id < 1:
            err_response['error'] = f'The JobRequest ID ({jr_id}) cannot be less than 1'
            return Response(err_response, status=status.HTTP_400_BAD_REQUEST)

        jr_list = models.JobRequest.objects.filter(id=jr_id)
        if len(jr_list) == 0:
            err_response['error'] = 'The JobRequest does not exist'
            return Response(err_response, status=status.HTTP_400_BAD_REQUEST)
        jr = jr_list[0]

        # JobRequest must have a Squonk Project value
        if not jr.squonk_project:
            err_response[
                'error'
            ] = f'The JobRequest ({jr_id}) has no Squonk Project value'
            return Response(err_response, status=status.HTTP_403_FORBIDDEN)

        # User must have access to the Job's Project.
        # If the user is not the owner of the Job, and there is a Project,
        # we then check the user has access to the given access ID.
        #
        # If the Job has no Project (Jobs created before this change will not have a Project)
        # or the user is the owner of the Job we skip this check.
        if user.id != jr.user.id:
            logger.info(
                '+ JobAccessView/GET Checking access to JobRequest %s for "%s" (%s)',
                jr_id,
                user.username,
                jr.squonk_project,
            )
            if not jr.project or not jr.project.title:
                logger.warning(
                    '+ JobAccessView/GET No Fragalysis Project (or title)'
                    ' for JobRequest %s - granting access',
                    jr_id,
                )
            else:
                # The project is the Job's access ID.
                # To check access we need this and the User's ID
                access_id = jr.project.id
                access_string = jr.project.title
                sq2a_common_params: CommonParams = CommonParams(
                    user_id=user.id,
                    access_id=access_id,
                    session_id=None,
                    target_id=None,
                )
                sq2a_run_job_params: RunJobParams = RunJobParams(
                    common=sq2a_common_params, job_spec=None, callback_url=None
                )
                sq2a_rv: Squonk2AgentRv = _SQ2A.can_run_job(sq2a_run_job_params)
                if not sq2a_rv.success:
                    err_response[
                        'error'
                    ] = f'Access to the Job for {access_id} ({access_string}) is denied. {sq2a_rv.msg}'
                    return Response(err_response, status=status.HTTP_403_FORBIDDEN)

            # All checks passed ... grant access for this user
            sq2a_access_params: AccessParams = AccessParams(
                username=user.username, project_uuid=jr.squonk_project
            )
            sqa_rv = _SQ2A.grant_access(sq2a_access_params)
            if not sqa_rv.success:
                err_response['error'] = f'Squonk failed to grant access ({sqa_rv.msg})'
                logger.warning('+ JobAccessView/GET error=%s', content['error'])
                return Response(
                    err_response, status=status.HTTP_500_INTERNAL_SERVER_ERROR
                )

        logger.info(
            '+ JobAccessView/GET Success for %s/"%s" on %s',
            jr_id,
            user.username,
            jr.squonk_project,
        )
        return Response(ok_response)


class ServiceStateView(View):
    def get(self, *args, **kwargs):
        """Poll external service status.

        Available services:
        - discourse
        - fragmentation_graph
        - ispyb
        - keycloak
        - squonk

        ENABLE_SERVICE_STATUS returns colon-separated id's for services
        """
        # Unused arguments
        del args, kwargs
        logger.debug("+ ServiceServiceState.State.get called")
        return JsonResponse(
            {"service_states": list(Service.data_manager.to_frontend())}
        )


class UploadMetadataView(ISPyBSafeQuerySet):
    serializer_class = serializers.MetadataUploadSerializer
    permission_class = [permissions.IsAuthenticated]
    http_method_names = ('post',)

    def get_view_name(self):
        return "Upload metadata"

    def create(self, request, *args, **kwargs):
        logger.info("+ UploadMetadataView.create called")
        del args, kwargs

        serializer = self.get_serializer_class()(data=request.data)
        if not serializer.is_valid():
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        logger.debug("Serializer validated_data=%s", serializer.validated_data)

        target_access_string = serializer.validated_data['target_access_string']
        target_name = serializer.validated_data['target']
        filename = serializer.validated_data['filename']

        try:
            project = models.Project.objects.get(title=target_access_string)
        except models.Project.DoesNotExist:
            return Response(
                {
                    "target_access_string": [
                        f"Project {target_access_string} does not exist"
                    ]
                },
                status=status.HTTP_400_BAD_REQUEST,
            )

        if project.title not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(
            request.user
        ):
            msg = f'User "{request.user.username}" is not a member of {target_access_string}'
            logger.warning(msg)
            content = {'message': msg}
            return Response(content, status=status.HTTP_404_NOT_FOUND)

        if settings.AUTHENTICATE_UPLOAD and not self.request.user.is_authenticated:
            return redirect(settings.LOGIN_URL)

        try:
            target = models.Target.objects.get(title=target_name, project=project)
        except models.Target.DoesNotExist:
            return Response(
                {"target": [f"Target {target_name} does not exist"]},
                status=status.HTTP_400_BAD_REQUEST,
            )

        # not celerifying it because seems fast enough
        errors = load_tags_from_file(filename=filename, target=target)
        if errors:
            return Response({"errors": errors}, status=status.HTTP_400_BAD_REQUEST)
        else:
            return Response({'success': True}, status=status.HTTP_200_OK)


class DownloadComputedSetView(ISPyBSafeQuerySet):
    queryset = models.ComputedSet.objects.all()
    filter_permissions = "target__project"
    serializer_class = serializers.ComputedSetDownloadSerializer
    permission_class = [permissions.IsAuthenticated]

    def get_view_name(self):
        return "Computed set download"

    def post(self, request, *args, **kwargs):
        logger.info("+ DownloadComputedSetView.create called")
        del args, kwargs

        logger.debug('data: %s', request.data)

        # If done like this, it's bypassing the validation..
        computed_set_name = request.data['name']
        try:
            computed_set = models.ComputedSet.objects.get(name=computed_set_name)
        except models.ComputedSet.DoesNotExist:
            return Response(
                {'error': f"ComputedSet '{computed_set_name}' not found"},
                status=status.HTTP_404_NOT_FOUND,
            )

        if (
            computed_set.target.project.title
            not in _ISPYB_SAFE_QUERY_SET.get_proposals_for_user(
                request.user, restrict_public_to_membership=False
            )
        ):
            return Response(
                {'error': "You have no access to the Project"},
                status=status.HTTP_403_FORBIDDEN,
            )

        zip_buffer = BytesIO()

        sdfs = models.ComputedSet.history.filter(
            written_sdf_filename__isnull=False,
        )
        pdbs = computed_set.computed_molecules.filter(pdb__isnull=True)

        # so now, get the file, and get the pdbs
        with zipfile.ZipFile(zip_buffer, 'a', zipfile.ZIP_DEFLATED) as ziparchive:
            for sdf in sdfs:
                sdf_file = Path(sdf.written_sdf_filename)
                if sdf_file.exists():
                    with open(sdf_file, 'rb') as contents:
                        ziparchive.writestr(str(sdf.submitted_sdf), contents.read())
                else:
                    ziparchive.writestr(f'{str(sdf.submitted_sdf)}_MISSING', r'')

            for f in pdbs:
                fpath = Path(settings.MEDIA_ROOT).joinpath(f.pdb_info.name)
                if fpath.is_file():
                    with open(fpath, 'rb') as contents:
                        ziparchive.writestr(f.get_filename(), contents.read())
                else:
                    ziparchive.writestr(f'{f.get_filename()}_MISSING', r'')

        response = HttpResponse(zip_buffer.getvalue(), content_type='application/zip')
        response['Content-Disposition'] = (
            'attachment; filename="%s"' % f'{computed_set.name}.zip'
        )
        response['Content-Length'] = zip_buffer.getbuffer().nbytes
        return response


class TokenView(APIView):
    def get(self, request, *args, **kwargs):
        """Return authentication token"""
        # Unused arguments
        del args, kwargs

        logger.debug("request.headers=%s", request.headers)

        if not request.user.is_authenticated:
            content: Dict[str, Any] = {
                'error': 'You need to be logged in to get a token'
            }
            return Response(content, status=status.HTTP_403_FORBIDDEN)

        try:
            sessionid = request.COOKIES['sessionid']
        except KeyError:
            return Response(
                {'error': 'Session could not be found, are you logged in?'},
                status=status.HTTP_403_FORBIDDEN,
            )

        return Response(
            {'sessionid': sessionid},
            status=status.HTTP_200_OK,
        )


class SiteObservationQualityStatusView(
    mixins.CreateModelMixin,
    mixins.UpdateModelMixin,
    ISPyBSafeQuerySet,
):
    queryset = models.SiteObservationQualityStatus.filter_manager.annotated_qs()
    serializer_class = serializers.SiteObservationQualityStatusSerializer
    filterset_class = filters.SiteObservationQualityStatusFilter
    filter_permissions = "site_observation__experiment__experiment_upload__project"