Remove xforms table from database, refactor project creation

src/backend/app/auth/auth_routes.py

@@ -250,6 +250,14 @@ async def refresh_token(
     request: Request, user_data: AuthUser = Depends(login_required)
 ):
     """Uses the refresh token to generate a new access token."""
+    if settings.DEBUG:
+        return JSONResponse(
+            status_code=HTTPStatus.OK,
+            content={
+                "token": "debugtoken",
+                **user_data.model_dump(),
+            },
+        )
     try:
         refresh_token = extract_refresh_token_from_cookie(request)
         if not refresh_token:

src/backend/app/central/central_crud.py

@@ -24,7 +24,6 @@
 from typing import Optional, Union
 
 import geojson
-from defusedxml import ElementTree
 from fastapi import HTTPException
 from loguru import logger as log
 from osm_fieldwork.OdkCentral import OdkAppUser, OdkForm, OdkProject
@@ -190,16 +189,15 @@ def create_odk_xform(
     odk_id: int,
     xform_data: BytesIO,
     odk_credentials: project_schemas.ODKCentralDecrypted,
-) -> str:
+) -> None:
     """Create an XForm on a remote ODK Central server.
 
     Args:
         odk_id (str): Project ID for ODK Central.
         xform_data (BytesIO): XForm data to set.
         odk_credentials (ODKCentralDecrypted): Creds for ODK Central.
 
-    Returns:
-        form_name (str): ODK Central form name for the API.
+    Returns: None
     """
     try:
         xform = get_odk_form(odk_credentials)
@@ -209,25 +207,7 @@ def create_odk_xform(
             status_code=500, detail={"message": "Connection failed to odk central"}
         ) from e
 
-    xform_id = xform.createForm(odk_id, xform_data, publish=True)
-    if not xform_id:
-        namespaces = {
-            "h": "http://www.w3.org/1999/xhtml",
-            "odk": "http://www.opendatakit.org/xforms",
-            "xforms": "http://www.w3.org/2002/xforms",
-        }
-        # Get the form id from the XML
-        root = ElementTree.fromstring(xform_data.getvalue())
-        xml_data = root.findall(".//xforms:data[@id]", namespaces)
-        extracted_name = "Not Found"
-        for dt in xml_data:
-            extracted_name = dt.get("id")
-        msg = f"Failed to create form on ODK Central: ({extracted_name})"
-        log.error(msg)
-        raise HTTPException(
-            status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg
-        ) from None
-    return xform_id
+    xform.createForm(odk_id, xform_data, publish=True)
 
 
 def delete_odk_xform(
@@ -323,7 +303,10 @@ async def read_and_test_xform(input_data: BytesIO) -> None:
         BytesIO: the converted XML representation of the XForm.
     """
     try:
-        log.debug("Parsing XLSForm --> XML data")
+        log.debug(
+            f"Parsing XLSForm --> XML data: input type {type(input_data)} | "
+            f"data length {input_data.getbuffer().nbytes}"
+        )
         # NOTE pyxform.xls2xform.convert returns a ConvertResult object
         return BytesIO(xform_convert(input_data).xform.encode("utf-8"))
     except Exception as e:
@@ -340,7 +323,7 @@ async def append_fields_to_user_xlsform(
     additional_entities: list[str] = None,
     task_count: int = None,
     existing_id: str = None,
-) -> BytesIO:
+) -> tuple[str, BytesIO]:
     """Helper to return the intermediate XLSForm prior to convert."""
     log.debug("Appending mandatory FMTM fields to XLSForm")
     return await append_mandatory_fields(
@@ -360,7 +343,7 @@ async def validate_and_update_user_xlsform(
     existing_id: str = None,
 ) -> BytesIO:
     """Wrapper to append mandatory fields and validate user uploaded XLSForm."""
-    updated_file_bytes = await append_fields_to_user_xlsform(
+    xform_id, updated_file_bytes = await append_fields_to_user_xlsform(
         xlsform,
         form_category=form_category,
         additional_entities=additional_entities,
@@ -899,12 +882,10 @@ async def get_appuser_token(
     xform_id: str,
     project_odk_id: int,
     odk_credentials: project_schemas.ODKCentralDecrypted,
-    db: Session,
 ):
     """Get the app user token for a specific project.
 
     Args:
-        db: The database session to use.
         odk_credentials: ODK credentials for the project.
         project_odk_id: The ODK ID of the project.
         xform_id: The ID of the XForm.

src/backend/app/central/central_routes.py

@@ -79,9 +79,8 @@ async def refresh_appuser_token(
     try:
         odk_credentials = await project_deps.get_odk_credentials(db, project_id)
         project_odk_id = project.odkid
-        db_xform = await project_deps.get_project_xform(db, project_id)
         odk_token = await central_crud.get_appuser_token(
-            db_xform.odk_form_id, project_odk_id, odk_credentials, db
+            project.odk_form_id, project_odk_id, odk_credentials, db
         )
         project.odk_token = odk_token
         db.commit()

src/backend/app/db/db_models.py

@@ -210,23 +210,6 @@ class DbXLSForm(Base):
     xls = cast(bytes, Column(LargeBinary))
 
 
-class DbXForm(Base):
-    """XForms linked per project.
-
-    TODO eventually we will support multiple forms per project.
-    TODO So the category field a stub until then.
-    TODO currently it's maintained under projects.xform_category.
-    """
-
-    __tablename__ = "xforms"
-    id = cast(int, Column(Integer, primary_key=True, autoincrement=True))
-    project_id = cast(
-        int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True)
-    )
-    odk_form_id = cast(str, Column(String))
-    category = cast(str, Column(String))
-
-
 class DbTaskHistory(Base):
     """Describes the history associated with a task."""
 
@@ -453,10 +436,8 @@ def tasks_bad(self):
 
     # XForm category specified
     xform_category = cast(str, Column(String))
-    # Linked XForms
-    forms = relationship(
-        DbXForm, backref="project_xform_link", cascade="all, delete, delete-orphan"
-    )
+    odk_form_id = cast(str, Column(String))
+    xlsform_content = cast(bytes, Column(LargeBinary))
 
     __table_args__ = (
         Index("idx_geometry", outline, postgresql_using="gist"),
@@ -486,13 +467,6 @@ def tasks_bad(self):
     odk_central_password = cast(str, Column(String))
     odk_token = cast(str, Column(String, nullable=True))
 
-    form_xls = cast(
-        bytes, Column(LargeBinary)
-    )  # XLSForm file if custom xls is uploaded
-    form_config_file = cast(
-        bytes, Column(LargeBinary)
-    )  # Yaml config file if custom xls is uploaded
-
     data_extract_type = cast(
         str, Column(String)
     )  # Type of data extract (Polygon or Centroid)
@@ -559,7 +533,11 @@ class DbSubmissionPhotos(Base):
     __tablename__ = "submission_photos"
 
     id = cast(int, Column(Integer, primary_key=True))
-    project_id = cast(int, Column(Integer))
-    task_id = cast(int, Column(Integer))
+    project_id = cast(
+        int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True)
+    )
+    task_id = cast(
+        int, Column(Integer, ForeignKey("tasks.id"), name="task_id", index=True)
+    )
     submission_id = cast(str, Column(String))
     s3_path = cast(str, Column(String))

src/backend/app/projects/project_crud.py

@@ -834,15 +834,13 @@ def flatten_dict(d, parent_key="", sep="_"):
 
 
 async def generate_odk_central_project_content(
-    project: db_models.DbProject,
+    project_odk_id: int,
+    project_odk_form_id: str,
     odk_credentials: project_schemas.ODKCentralDecrypted,
     xlsform: BytesIO,
     task_extract_dict: dict[int, geojson.FeatureCollection],
-    db: Session,
 ) -> str:
     """Populate the project in ODK Central with XForm, Appuser, Permissions."""
-    project_odk_id = project.odkid
-
     # The ODK Dataset (Entity List) must exist prior to main XLSForm
     entities_list = await central_crud.task_geojson_dict_to_entity_values(
         task_extract_dict
@@ -861,33 +859,16 @@ async def generate_odk_central_project_content(
 
     # Upload survey XForm
     log.info("Uploading survey XForm to ODK Central")
-    xform_id = central_crud.create_odk_xform(
+    central_crud.create_odk_xform(
         project_odk_id,
         xform,
         odk_credentials,
     )
 
-    sql = text(
-        """
-        INSERT INTO xforms (
-            project_id, odk_form_id, category
-        )
-        VALUES (
-            :project_id, :xform_id, :category
-        )
-        """
-    )
-    db.execute(
-        sql,
-        {
-            "project_id": project.id,
-            "xform_id": xform_id,
-            "category": project.xform_category,
-        },
-    )
-    db.commit()
     return await central_crud.get_appuser_token(
-        xform_id, project_odk_id, odk_credentials, db
+        project_odk_form_id,
+        project_odk_id,
+        odk_credentials,
     )
 
 
@@ -929,13 +910,15 @@ async def generate_project_files(
 
         # Get ODK Project ID
         project_odk_id = project.odkid
+        project_xlsform = project.xlsform_content
+        project_odk_form_id = project.odk_form_id
 
         encrypted_odk_token = await generate_odk_central_project_content(
-            project,
+            project_odk_id,
+            project_odk_form_id,
             odk_credentials,
-            BytesIO(project.form_xls),
+            BytesIO(project_xlsform),
             task_extract_dict,
-            db,
         )
         log.debug(
             f"Setting odk token for FMTM project ({project_id}) "
@@ -1488,9 +1471,8 @@ async def get_dashboard_detail(
     """Get project details for project dashboard."""
     odk_central = await project_deps.get_odk_credentials(db, project.id)
     xform = central_crud.get_odk_form(odk_central)
-    db_xform = await project_deps.get_project_xform(db, project.id)
 
-    submission_meta_data = xform.getFullDetails(project.odkid, db_xform.odk_form_id)
+    submission_meta_data = xform.getFullDetails(project.odkid, project.odk_form_id)
     project.total_submission = submission_meta_data.get("submissions", 0)
     project.last_active = submission_meta_data.get("lastSubmission")
 

src/backend/app/projects/project_deps.py

@@ -98,28 +98,3 @@ async def get_odk_credentials(db: Session, project_id: int):
         odk_central_user=user,
         odk_central_password=password,
     )
-
-
-async def get_project_xform(db, project_id):
-    """Retrieve the transformation associated with a specific project.
-
-    Args:
-        db: Database connection object.
-        project_id: The ID of the project to retrieve the transformation for.
-
-    Returns:
-        The transformation record associated with the specified project.
-
-    Raises:
-        None
-    """
-    sql = text(
-        """
-        SELECT * FROM xforms
-        WHERE project_id = :project_id;
-    """
-    )
-
-    result = db.execute(sql, {"project_id": project_id})
-    db_xform = result.first()
-    return db_xform

src/backend/app/projects/project_routes.py

@@ -666,10 +666,14 @@ async def validate_form(
 ):
     """Basic validity check for uploaded XLSForm.
 
-    Does not append all addition values to make this a valid FMTM form for mapping.
+    Parses the form using ODK pyxform to check that it is valid.
+
+    If the `debug` param is used, the form is returned for inspection.
+    NOTE that this debug form has additional fields appended and should
+        not be used for FMTM project creation.
     """
     if debug:
-        updated_form = await central_crud.append_fields_to_user_xlsform(
+        xform_id, updated_form = await central_crud.append_fields_to_user_xlsform(
             xlsform,
             task_count=1,  # NOTE this must be included to append task_filter choices
         )
@@ -678,14 +682,17 @@ async def validate_form(
             media_type=(
                 "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet"
             ),
-            headers={"Content-Disposition": "attachment; filename=updated_form.xlsx"},
+            headers={"Content-Disposition": f"attachment; filename={xform_id}.xlsx"},
         )
     else:
         await central_crud.validate_and_update_user_xlsform(
             xlsform,
             task_count=1,  # NOTE this must be included to append task_filter choices
         )
-        return Response(status_code=HTTPStatus.OK)
+        return JSONResponse(
+            status_code=HTTPStatus.OK,
+            content={"message": "Your form is valid"},
+        )
 
 
 @router.post("/{project_id}/generate-project-data")
@@ -752,14 +759,21 @@ async def generate_files(
         with open(xlsform_path, "rb") as f:
             xlsform = BytesIO(f.read())
 
-    project_xlsform = await central_crud.append_fields_to_user_xlsform(
+    xform_id, project_xlsform = await central_crud.append_fields_to_user_xlsform(
         xlsform=xlsform,
         form_category=form_category,
         task_count=task_count,
         additional_entities=additional_entities,
     )
     # Write XLS form content to db
-    project.form_xls = project_xlsform.getvalue()
+    xlsform_bytes = project_xlsform.getvalue()
+    if not xlsform_bytes:
+        raise HTTPException(
+            status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+            detail="There was an error with the XLSForm!",
+        )
+    project.odk_form_id = xform_id
+    project.xlsform_content = xlsform_bytes
     db.commit()
 
     # Create task in db and return uuid
@@ -979,7 +993,7 @@ async def download_form(
         "Content-Disposition": f"attachment; filename={project.id}_xlsform.xlsx",
         "Content-Type": "application/media",
     }
-    return Response(content=project.form_xls, headers=headers)
+    return Response(content=project.xlsform_content, headers=headers)
 
 
 @router.post("/update-form")
@@ -1019,7 +1033,7 @@ async def update_project_form(
     )
 
     # Commit changes to db
-    project.form_xls = xlsform.getvalue()
+    project.xlsform_content = xlsform.getvalue()
     db.commit()
 
     return project