Reduce DB contention: skip project UPDATE when registering new files

SPRINTLOG.md

@@ -660,3 +660,4 @@ _Nothing merged during this sprint_
 
 - Update dependency sass to v1.98.0([#1799]https://github.com/ScilifelabDataCentre/dds_web/pull/1799)
 - Update dependency cryptography to v50 [SECURITY] ([#1857]https://github.com/ScilifelabDataCentre/dds_web/pull/1857)
+- Reduce DB contention: skip project UPDATE when registering new files ([#1813]https://github.com/ScilifelabDataCentre/dds_web/pull/1813)

dds_web/api/__init__.py

@@ -67,6 +67,9 @@ def add_resources(api):
     api.add_resource(project.ProjectAccess, "/proj/access", endpoint="project_access")
     api.add_resource(project.ProjectBusy, "/proj/busy", endpoint="project_busy")
     api.add_resource(project.ProjectInfo, "/proj/info", endpoint="project_info")
+    api.add_resource(
+        project.ProjectUploadComplete, "/proj/upload/complete", endpoint="project_upload_complete"
+    )
 
     # User management ################################################################ User management #
     api.add_resource(user.RetrieveUserInfo, "/user/info", endpoint="user_info")

dds_web/api/files.py

@@ -87,10 +87,11 @@ def post(self):
         # Verify that project has correct status for upload
         check_eligibility_for_upload(status=project.current_status)
 
-        # Create new files
+        # Create new files (new_file is not attached via project.files so add explicitly)
         new_file = file_schemas.NewFileSchema().load(
             {**flask.request.get_json(silent=True), "project": project.public_id}
         )
+        db.session.add(new_file)
 
         try:
             db.session.commit()
@@ -176,11 +177,11 @@ def put(self):
                 size_stored=file_info.get("size_processed"),
                 time_uploaded=new_timestamp,
                 active_file=existing_file.id,
-                project_id=project,
             )
 
-            # Update foreign keys and relationships
-            project.file_versions.append(new_version)
+            # Set FK directly so we do not modify the project row (avoids UPDATE on projects
+            # and reduces lock contention during concurrent PUT /file/new).
+            new_version.project_id = project.id
             existing_file.versions.append(new_version)
 
             db.session.add(new_version)

dds_web/api/project.py

@@ -30,7 +30,7 @@
     json_required,
     logging_bind_request,
 )
-from dds_web.api.files import check_eligibility_for_deletion
+from dds_web.api.files import check_eligibility_for_deletion, check_eligibility_for_upload
 from dds_web.api.schemas import project_schemas, user_schemas
 from dds_web.api.user import AddUser
 from dds_web.database import models
@@ -1313,3 +1313,37 @@ def put(self):
         }
 
         return return_message
+
+
+class ProjectUploadComplete(flask_restful.Resource):
+    """Update ``date_updated`` / ``last_updated_by`` once after a batch upload.
+
+    The CLI calls this at the end of ``dds data put`` when at least one file was
+    registered in the database, so project metadata stays accurate without
+    updating the project row on every ``POST /file/new``.
+    """
+
+    @auth.login_required(role=["Unit Admin", "Unit Personnel"])
+    @logging_bind_request
+    @handle_validation_errors
+    def post(self):
+        project = project_schemas.ProjectRequiredSchema().load(flask.request.args)
+        check_eligibility_for_upload(status=project.current_status)
+        # Dirty the row so the before_update listener in models.py sets date_updated and last_updated_by.
+        project.date_updated = dds_web.utils.current_time()
+        try:
+            db.session.commit()
+        except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err:
+            flask.current_app.logger.debug(err)
+            db.session.rollback()
+            raise DatabaseError(
+                message=str(err),
+                alt_message="Failed to update project timestamp after upload"
+                + (
+                    ": Database malfunction."
+                    if isinstance(err, sqlalchemy.exc.OperationalError)
+                    else "."
+                ),
+            ) from err
+
+        return {"message": "Project upload timestamp updated."}

dds_web/api/schemas/file_schemas.py

@@ -140,9 +140,10 @@ def return_items(self, data, **kwargs):
         )
 
         project = data.get("project_row")
-        # Update foreign keys
-        project.file_versions.append(new_version)
-        project.files.append(new_file)
+        # Set FKs directly so we do not modify the project row (avoids UPDATE on projects
+        # and reduces lock contention during concurrent POST /file/new).
+        new_file.project_id = project.id
+        new_version.project_id = project.id
         new_file.versions.append(new_version)
 
         return new_file

tests/__init__.py

@@ -204,6 +204,7 @@ class DDSEndpoint:
     PROJECT_BUSY = BASE_ENDPOINT + "/proj/busy"
     PROJECT_BUSY_ANY = BASE_ENDPOINT + "/proj/busy/any"
     PROJECT_INFO = BASE_ENDPOINT + "/proj/info"
+    PROJ_UPLOAD_COMPLETE = BASE_ENDPOINT + "/proj/upload/complete"
 
     # Listing urls
     LIST_PROJ = BASE_ENDPOINT + "/proj/list"

tests/test_files_new.py

@@ -418,6 +418,35 @@ def test_new_file_database_error(client):
     assert rollback.called
 
 
+def test_new_file_post_does_not_update_project_timestamp(client):
+    """POST /file/new must not update project.date_updated or last_updated_by."""
+    import freezegun
+    import datetime
+
+    project_1 = project_row(project_id="file_testing_project")
+    assert project_1
+
+    frozen_time = datetime.datetime(2000, 1, 1, 12, 0, 0)
+    with freezegun.freeze_time(frozen_time):
+        project_1.date_updated = frozen_time
+        db.session.commit()
+
+    original_date_updated = project_1.date_updated
+    original_last_updated_by = project_1.last_updated_by
+
+    response = client.post(
+        tests.DDSEndpoint.FILE_NEW,
+        headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client),
+        query_string={"project": "file_testing_project"},
+        json=FIRST_NEW_FILE,
+    )
+    assert response.status_code == http.HTTPStatus.OK
+
+    db.session.refresh(project_1)
+    assert project_1.date_updated == original_date_updated
+    assert project_1.last_updated_by == original_last_updated_by
+
+
 def test_new_file(client):
     """Add and overwrite file to database."""
 
@@ -462,6 +491,44 @@ def test_new_file(client):
     assert f"File '{updated_file['name']}' updated in db." in response.json["message"]
 
 
+def test_new_file_put_version_linked_to_file_and_project(client):
+    """PUT /file/new creates a Version row tied to both the file and the project."""
+    project_1 = project_row(project_id="file_testing_project")
+    assert project_1
+
+    # First add the file
+    response = client.post(
+        tests.DDSEndpoint.FILE_NEW,
+        headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client),
+        query_string={"project": "file_testing_project"},
+        json=FIRST_NEW_FILE,
+    )
+    assert response.status_code == http.HTTPStatus.OK
+
+    # Overwrite the file via PUT
+    updated_file = {**FIRST_NEW_FILE, "size": 1200, "size_processed": 600}
+    response = client.put(
+        tests.DDSEndpoint.FILE_NEW,
+        headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client),
+        query_string={"project": "file_testing_project"},
+        json=updated_file,
+    )
+    assert response.status_code == http.HTTPStatus.OK
+
+    # Fetch the file row and verify the new version is linked correctly
+    file_row = models.File.query.filter_by(
+        name=FIRST_NEW_FILE["name"], project_id=project_1.id
+    ).one_or_none()
+    assert file_row is not None
+
+    new_version = models.Version.query.filter_by(
+        active_file=file_row.id, time_deleted=None
+    ).one_or_none()
+    assert new_version is not None
+    assert new_version.active_file == file_row.id
+    assert new_version.project_id == project_1.id
+
+
 def test_update_nonexistent_file(client):
     """Try to update a non existent file"""
     response = client.put(

tests/test_init.py

@@ -316,6 +316,15 @@ def test_block_if_maintenance_active_none_approved_users(client: flask.testing.F
         assert response.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE
         assert response.json and response.json.get("message") == "Maintenance of DDS is ongoing."
 
+        # ProjectUploadComplete - "/proj/upload/complete"
+        # No project param needed — maintenance middleware blocks before the handler runs.
+        response = client.post(
+            DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+            headers=token,
+        )
+        assert response.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE
+        assert response.json and response.json.get("message") == "Maintenance of DDS is ongoing."
+
         # RetrieveUserInfo - "/user/info"
         response = client.get(
             DDSEndpoint.USER_INFO,
@@ -630,6 +639,14 @@ def test_block_if_maintenance_active_superadmin_ok(client: flask.testing.FlaskCl
     )
     assert response.status_code == http.HTTPStatus.BAD_REQUEST
 
+    # ProjectUploadComplete - "/proj/upload/complete"
+    # No project param needed — role check fires before schema validation.
+    response = client.post(
+        DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+        headers=token,
+    )
+    assert response.status_code == http.HTTPStatus.FORBIDDEN
+
     # RetrieveUserInfo - "/user/info"
     response = client.get(
         DDSEndpoint.USER_INFO,

tests/test_project_upload.py

@@ -0,0 +1,147 @@
+# IMPORTS ################################################################################ IMPORTS #
+
+# Standard library
+import http
+import datetime
+
+# Installed
+import freezegun
+import sqlalchemy
+from unittest.mock import patch
+
+# Own
+from dds_web import db
+import tests
+from tests.test_files_new import project_row
+
+
+# TESTS #################################################################################### TESTS #
+
+# ProjectUploadComplete - "/proj/upload/complete"
+
+
+def test_proj_upload_complete_updates_timestamp(client):
+    """POST /proj/upload/complete refreshes date_updated and last_updated_by."""
+    project_1 = project_row(project_id="file_testing_project")
+    assert project_1
+
+    frozen_before = datetime.datetime(2000, 1, 1, 12, 0, 0)
+    frozen_after = datetime.datetime(2000, 1, 2, 12, 0, 0)
+
+    with freezegun.freeze_time(frozen_before):
+        project_1.date_updated = frozen_before
+        db.session.commit()
+
+    token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client)
+
+    with freezegun.freeze_time(frozen_after):
+        response = client.post(
+            tests.DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+            headers=token,
+            query_string={"project": "file_testing_project"},
+        )
+    assert response.status_code == http.HTTPStatus.OK
+    assert response.json.get("message") == "Project upload timestamp updated."
+
+    db.session.refresh(project_1)
+    assert project_1.date_updated == frozen_after
+    assert project_1.last_updated_by == "unitadmin"
+
+
+def test_proj_upload_complete_unit_personnel_allowed(client):
+    """Unit Personnel (non-admin unit user) can call POST /proj/upload/complete."""
+    response = client.post(
+        tests.DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+        headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client),
+        query_string={"project": "file_testing_project"},
+    )
+    assert response.status_code == http.HTTPStatus.OK
+
+
+def test_proj_upload_complete_unauthorized_roles_denied(client):
+    """Researcher and Project Owner cannot call POST /proj/upload/complete."""
+    for role in ("researcher", "projectowner"):
+        response = client.post(
+            tests.DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+            headers=tests.UserAuth(tests.USER_CREDENTIALS[role]).token(client),
+            query_string={"project": "file_testing_project"},
+        )
+        assert (
+            response.status_code == http.HTTPStatus.FORBIDDEN
+        ), f"Expected 403 for role '{role}', got {response.status_code}"
+
+
+def test_proj_upload_complete_missing_project(client):
+    """POST /proj/upload/complete returns 400 when project query param is missing."""
+    response = client.post(
+        tests.DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+        headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client),
+    )
+    assert response.status_code == http.HTTPStatus.BAD_REQUEST
+    assert response.json.get("project", {}).get("message") == "Project ID required."
+
+
+def test_proj_upload_complete_db_failure(client):
+    """POST /proj/upload/complete returns 500 on database error."""
+    token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client)
+
+    with patch("dds_web.db.session.commit") as mock_commit:
+        mock_commit.side_effect = sqlalchemy.exc.SQLAlchemyError()
+
+        response = client.post(
+            tests.DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+            headers=token,
+            query_string={"project": "file_testing_project"},
+        )
+    assert response.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR
+    assert "Failed to update project timestamp after upload" in response.json["message"]
+
+
+def test_proj_upload_complete_no_update_if_available(client, boto3_session):
+    """POST /proj/upload/complete returns 400 when project status is Available."""
+    token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client)
+
+    # Move project to Available
+    response = client.post(
+        tests.DDSEndpoint.PROJECT_STATUS,
+        headers=token,
+        query_string={"project": "file_testing_project"},
+        json={"new_status": "Available"},
+    )
+    assert response.status_code == http.HTTPStatus.OK
+
+    response = client.post(
+        tests.DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+        headers=token,
+        query_string={"project": "file_testing_project"},
+    )
+    assert response.status_code == http.HTTPStatus.BAD_REQUEST
+
+
+def test_proj_upload_complete_no_update_if_expired(client, boto3_session, mock_queue_redis):
+    """POST /proj/upload/complete returns 400 when project status is Expired."""
+    token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client)
+
+    # Move project to Available then Expired
+    response = client.post(
+        tests.DDSEndpoint.PROJECT_STATUS,
+        headers=token,
+        query_string={"project": "file_testing_project"},
+        json={"new_status": "Available"},
+    )
+    assert response.status_code == http.HTTPStatus.OK
+
+    response = client.post(
+        tests.DDSEndpoint.PROJECT_STATUS,
+        headers=token,
+        query_string={"project": "file_testing_project"},
+        json={"new_status": "Expired"},
+    )
+    assert response.status_code == http.HTTPStatus.OK
+
+    response = client.post(
+        tests.DDSEndpoint.PROJ_UPLOAD_COMPLETE,
+        headers=token,
+        query_string={"project": "file_testing_project"},
+    )
+    assert response.status_code == http.HTTPStatus.BAD_REQUEST

tests/tests_v3/__init__.py

@@ -205,6 +205,7 @@ class DDSEndpoint:
     PROJECT_BUSY = BASE_ENDPOINT + "/proj/busy"
     PROJECT_BUSY_ANY = BASE_ENDPOINT + "/proj/busy/any"
     PROJECT_INFO = BASE_ENDPOINT + "/proj/info"
+    PROJ_UPLOAD_COMPLETE = BASE_ENDPOINT + "/proj/upload/complete"
 
     # Listing urls
     LIST_PROJ = BASE_ENDPOINT + "/proj/list"