Update SIM processing workflow

src/murfey/client/contexts/sim.py

@@ -35,11 +35,7 @@ def post_transfer(
         # These have no extensions, and end with one of the listed suffixes
         if not transferred_file.suffix and transferred_file.stem.endswith(
             (
-                # Fluorescent SIM raw data files end as follows
-                "_BR",
-                "_BFR",
-                "_GR",
-                "_GFR",
+                # Only SIM raw data files ending with '_FL' should be processed
                 "_BR_FL",
                 "_BFR_FL",
                 "_GR_FL",
@@ -62,7 +58,7 @@ def post_transfer(
             capture_post(
                 base_url=str(environment.url.geturl()),
                 router_name="workflow_sim.router",
-                function_name="request_sim_processing",
+                function_name="request_sim_reconstruction",
                 token=self._token,
                 instrument_name=environment.instrument_name,
                 data={

src/murfey/server/api/workflow_sim.py

@@ -1,13 +1,18 @@
 import json
 import logging
 from pathlib import Path
+from typing import Any
 
 from fastapi import APIRouter, Depends
 from pydantic import BaseModel
+from sqlmodel import Session as SQLModelSession, select
 
+import murfey.util.db as MurfeyDB
 from murfey.server import _transport_object
 from murfey.server.api.auth import validate_instrument_token
+from murfey.server.murfey_db import murfey_db
 from murfey.util import sanitise_path
+from murfey.util.config import get_machine_config
 
 logger = logging.getLogger("murfey.server.api.workflow_sim")
 
@@ -22,22 +27,99 @@ class SIMDataFile(BaseModel):
     file: Path
 
 
-@router.post("/sessions/{session_id}/process_data")
-def request_sim_processing(session_id: int, sim_data: SIMDataFile):
+@router.post("/sessions/{session_id}/sim_recon")
+def request_sim_reconstruction(
+    session_id: int,
+    sim_data: SIMDataFile,
+    murfey_db: SQLModelSession = murfey_db,
+):
     if _transport_object is None:
         logger.error("No TransportManager object was set up")
         return None
 
+    # Load instrument and visit information based on session
+    try:
+        murfey_session = murfey_db.exec(
+            select(MurfeyDB.Session).where(MurfeyDB.Session.id == session_id)
+        ).one()
+        instrument_name = murfey_session.instrument_name
+        visit_name = murfey_session.visit
+    except Exception:
+        logger.error("Error querying session information from database", exc_info=True)
+        return None
+
+    # Load PySIMRecon values from the machine config
+    try:
+        machine_config = get_machine_config(instrument_name)[instrument_name]
+        pysimrecon_config: dict[str, dict[str, Any]] | None = (
+            machine_config.calibrations.get("pysimrecon_config")
+        )
+        if not pysimrecon_config:
+            # If no calibration was provided, use defaults
+            # Values provided on 2026-07-16
+            pysimrecon_config = {
+                "blue": {
+                    "wavelength": 452,
+                    "ls": 0.330,
+                    "beaddiam": 0.220,
+                },
+                "green": {
+                    "wavelength": 525,
+                    "ls": 0.394,
+                },
+                "red": {
+                    "wavelength": 605,
+                    "ls": 0.451,
+                },
+                "far_red": {
+                    "wavelength": 655,
+                    "ls": 0.521,
+                },
+            }
+            logger.warning(
+                f"No PySIMRecon configuration found for {instrument_name}; "
+                f"using known defaults \n{json.dumps(pysimrecon_config, indent=2)}"
+            )
+    except Exception:
+        logger.error("Error loading machine config from database", exc_info=True)
+        return None
+
     # Construct message and submit it to 'processing_recipe'
     logger.info(
         f"Submitting request to process the cryoSIM file {sanitise_path(sim_data.file)}"
     )
+    # Construct the output directory for the PySIMRecon outputs to be saved to
+    try:
+        visit_idx = sim_data.file.parts.index(visit_name)
+        raw_dir = Path(
+            "/".join(
+                ""
+                if part == "/"  # Replace root "/" with "" for Linux paths
+                else part
+                for part in sim_data.file.parts[: visit_idx + 2]
+            )
+        )
+        output_dir = (
+            raw_dir.parent / "processed" / sim_data.file.parent.relative_to(raw_dir)
+        )
+    except Exception:
+        logger.error(
+            "Could not determine the output directory to save the cryoSIM file "
+            f"{sanitise_path(sim_data.file)} to"
+        )
+        return None
     recipe = {
-        "recipes": ["sim-process-data"],
+        "recipes": ["sim-reconstruction"],
         "parameters": {
-            # Job parameters
-            "session_id": session_id,
+            # PySIMRecon parameters
             "file": f"{str(sim_data.file)}",
+            "output_dir": str(output_dir),
+            "blue_params": str(pysimrecon_config["blue"]),
+            "green_params": str(pysimrecon_config["green"]),
+            "red_params": str(pysimrecon_config["red"]),
+            "far_red_params": str(pysimrecon_config["far_red"]),
+            # Return message
+            "session_id": session_id,
             "feedback_queue": _transport_object.feedback_queue,
         },
     }
@@ -46,6 +128,6 @@ def request_sim_processing(session_id: int, sim_data: SIMDataFile):
         f"{json.dumps(recipe, indent=2, default=str)}"
     )
     # Disabled for now; will submit message once recipe and service have been set up
-    # _transport_object.send(
-    #     queue="processing_recipe", message=recipe, new_connection=True
-    # )
+    _transport_object.send(
+        queue="processing_recipe", message=recipe, new_connection=True
+    )

src/murfey/util/route_manifest.yaml

@@ -1453,8 +1453,8 @@ murfey.server.api.workflow_fib.router:
     methods:
       - POST
 murfey.server.api.workflow_sim.router:
-  - path: /workflow/sim/sessions/{session_id}/process_data
-    function: request_sim_processing
+  - path: /workflow/sim/sessions/{session_id}/sim_recon
+    function: request_sim_reconstruction
     path_params:
       - name: session_id
         type: int

tests/client/contexts/test_sim.py

@@ -29,6 +29,7 @@ def sim_data(visit_dir: Path):
         "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR",
         "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR",
         "raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR",
+        # To be processed
         "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR_FL",
         "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR_FL",
         "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR_FL",
@@ -132,7 +133,7 @@ def test_post_transfer(
     destination_files = [
         destination_dir / file.relative_to(visit_dir)
         for file in sim_data
-        if not file.stem.endswith("_BF")
+        if file.stem.endswith("_FL")
     ]
 
     # Mock the functions used in 'post_transfer'
@@ -162,7 +163,7 @@ def test_post_transfer(
         mock_logger.warning.assert_called_with(f"No source found for file {file}")
     else:
         for src, dst in zip(sim_data, [Path(""), *destination_files]):
-            if src.stem.endswith("_BF"):
+            if not src.stem.endswith("_FL"):
                 continue
             else:
                 mock_get_source.assert_any_call(src, mock_environment)
@@ -175,7 +176,7 @@ def test_post_transfer(
                 mock_capture_post.assert_any_call(
                     base_url=mock.ANY,
                     router_name="workflow_sim.router",
-                    function_name="request_sim_processing",
+                    function_name="request_sim_reconstruction",
                     token=context._token,
                     instrument_name=instrument_name,
                     data={
@@ -184,4 +185,4 @@ def test_post_transfer(
                     # Endpoint kwargs
                     session_id=session_id,
                 )
-        assert mock_capture_post.call_count == len(sim_data) - 1
+        assert mock_capture_post.call_count == 4

tests/server/api/test_workflow_sim.py

@@ -5,20 +5,109 @@
 import pytest
 from pytest_mock import MockerFixture
 
-from murfey.server.api.workflow_sim import SIMDataFile, request_sim_processing
+from murfey.server.api.workflow_sim import SIMDataFile, request_sim_reconstruction
+from murfey.util import sanitise_path
+from murfey.util.config import MachineConfig
 
+# Global variables
+instrument_name = "sim"
+visit_name = "cm12345-6"
+session_id = 1
 
-@pytest.mark.parametrize("has_transport_object", (True, False))
-def test_request_sim_processing(
-    mocker: MockerFixture, tmp_path: Path, has_transport_object: bool
+
+@pytest.fixture
+def visit_dir(tmp_path: Path):
+    visit_dir = tmp_path / "data" / "2020" / visit_name
+    visit_dir.mkdir(parents=True, exist_ok=True)
+    return visit_dir
+
+
+@pytest.mark.parametrize(
+    "test_params",
+    (  # Transport object | DB query success | Machine config found | PySIMRecon config found | Output dir found
+        # Successful case
+        (True, True, True, True, True),
+        (True, True, True, False, True),  # No PySIMRecon config
+        # Failure cases
+        (False, True, True, True, True),  # No transport object
+        (True, False, True, True, True),  # DB query failed
+        (True, True, False, True, True),  # Machine config error
+        (True, True, True, True, False),  # Incorrect output dir
+    ),
+)
+def test_request_sim_reconstruction(
+    mocker: MockerFixture,
+    tmp_path: Path,
+    visit_dir: Path,
+    test_params: tuple[bool, bool, bool, bool, bool],
 ):
-    # Set up the variables
-    session_id = 1
-    sim_data = SIMDataFile(**{"file": str(tmp_path / "dummy")})
+    # Unpack test params
+    (
+        has_transport_object,
+        db_query_success,
+        machine_config_found,
+        pysimrecon_configured,
+        output_dir_success,
+    ) = test_params
+
+    # Set up the test file and output directory
+    test_file = (
+        visit_dir / "raw" / "grid_1" / "test_file"
+        if output_dir_success
+        else tmp_path / "dummy"  # Provide incorrect file path
+    )
+    sim_data = SIMDataFile(**{"file": str(test_file)})
+    output_dir = visit_dir / "processed" / "grid_1"
+    output_dir.mkdir(parents=True, exist_ok=True)
 
     # Mock the logger
     mock_logger = mocker.patch("murfey.server.api.workflow_sim.logger")
 
+    # Mock the Murfey DB
+    mock_murfey_session = MagicMock(
+        instrument_name=instrument_name,
+        visit=visit_name,
+    )
+    mock_db = MagicMock()
+    if db_query_success:
+        mock_db.exec.return_value.one.return_value = mock_murfey_session
+    else:
+        mock_db.exec.return_value.one.side_effect = Exception("Something went wrong")
+
+    # Mock the machine config
+    blue_params = {
+        "wavelength": 452,
+        "ls": 0.123 if pysimrecon_configured else 0.330,
+        "beaddiam": 0.220,
+    }
+    green_params = {
+        "wavelength": 525,
+        "ls": 0.234 if pysimrecon_configured else 0.394,
+    }
+    red_params = {
+        "wavelength": 605,
+        "ls": 0.345 if pysimrecon_configured else 0.451,
+    }
+    far_red_params = {
+        "wavelength": 655,
+        "ls": 0.456 if pysimrecon_configured else 0.521,
+    }
+    pysimrecon_config = {
+        "blue": blue_params,
+        "green": green_params,
+        "red": red_params,
+        "far_red": far_red_params,
+    }
+    machine_config = MachineConfig(
+        calibrations={"pysimrecon_config": pysimrecon_config}
+        if pysimrecon_configured
+        else {},
+    )
+    mocker.patch(
+        "murfey.server.api.workflow_sim.get_machine_config",
+        return_value={instrument_name: machine_config} if machine_config_found else {},
+    )
+
     # Mock the transport object
     if has_transport_object:
         mock_transport_object = MagicMock()
@@ -34,27 +123,48 @@ def test_request_sim_processing(
         )
 
     # Run the function and check that the expected calls were made
-    request_sim_processing(
-        session_id=session_id,
-        sim_data=sim_data,
+    request_sim_reconstruction(
+        session_id=session_id, sim_data=sim_data, murfey_db=mock_db
     )
 
     # Check that the expected calls were made
-    if has_transport_object:
+    # The parameters are toggled 'False' one at a time
+    if not has_transport_object:
+        mock_logger.error.assert_called_with("No TransportManager object was set up")
+    elif not db_query_success:
+        mock_logger.error.assert_called_with(
+            "Error querying session information from database", exc_info=True
+        )
+        mock_transport_object.send.assert_not_called()
+    elif not machine_config_found:
+        mock_logger.error.assert_called_with(
+            "Error loading machine config from database", exc_info=True
+        )
+        mock_transport_object.send.assert_not_called()
+    elif not output_dir_success:
+        mock_logger.error.assert_called_with(
+            "Could not determine the output directory to save the cryoSIM file "
+            f"{sanitise_path(sim_data.file)} to"
+        )
+        mock_transport_object.send.assert_not_called()
+    else:
         recipe = {
-            "recipes": ["sim-process-data"],
+            "recipes": ["sim-reconstruction"],
             "parameters": {
+                "file": f"{str(sim_data.file)}",
+                "output_dir": str(output_dir),
+                "blue_params": str(pysimrecon_config["blue"]),
+                "green_params": str(pysimrecon_config["green"]),
+                "red_params": str(pysimrecon_config["red"]),
+                "far_red_params": str(pysimrecon_config["far_red"]),
                 "session_id": session_id,
-                "file": f"{sim_data.file}",
                 "feedback_queue": "dummy",
             },
         }
         mock_logger.debug.assert_called_with(
             "Will submit the following message to 'processing_recipe':\n"
             f"{json.dumps(recipe, indent=2, default=str)}"
         )
-        # mock_transport_object.send.assert_called_with(
-        #     queue="processing_recipe", message=recipe, new_connection=True
-        # )
-    else:
-        mock_logger.error.assert_called_with("No TransportManager object was set up")
+        mock_transport_object.send.assert_called_with(
+            queue="processing_recipe", message=recipe, new_connection=True
+        )