Add db components from ispyb for processing outputs

src/murfey/server/api/workflow.py

@@ -443,6 +443,7 @@ async def request_spa_preprocessing(
             db.add(feedback_params)
         movie = Movie(
             murfey_id=murfey_ids[0],
+            data_collection_id=detached_ids[1],
             path=proc_file.path,
             image_number=proc_file.image_number,
             tag=proc_file.tag,
@@ -695,6 +696,17 @@ async def request_tomography_preprocessing(
                 0
             ].eer_fractionation_file
 
+        movie = Movie(
+            murfey_id=murfey_ids[0],
+            data_collection_id=dcid,
+            path=proc_file.path,
+            image_number=proc_file.image_number,
+            tag=proc_file.tag,
+        )
+        db.add(movie)
+        db.commit()
+        db.close()
+
         zocalo_message: dict = {
             "recipes": [recipe_name],
             "parameters": {

src/murfey/server/feedback.py

@@ -1488,6 +1488,7 @@ def _flush_tomography_preprocessing(message: dict, _db):
             p.parent.mkdir(parents=True)
         movie = db.Movie(
             murfey_id=murfey_ids[0],
+            data_collection_id=detached_ids[1],
             path=f.file_path,
             image_number=f.image_number,
             tag=f.tag,

src/murfey/util/db.py

@@ -4,11 +4,22 @@
 """
 
 from datetime import datetime
-from typing import List, Optional
+from typing import TYPE_CHECKING, List, Optional
 
 import sqlalchemy
 from sqlmodel import Field, Relationship, SQLModel, create_engine
 
+if TYPE_CHECKING:
+    from murfey.util.processing_db import (
+        CTF,
+        MotionCorrection,
+        ParticleClassificationGroup,
+        ParticlePicker,
+        RelativeIceThickness,
+        TiltImageAlignment,
+        Tomogram,
+    )
+
 """
 GENERAL
 """
@@ -429,6 +440,14 @@
             sa_relationship_kwargs={"cascade": "delete"},
         )
     )
+    grid_squares: List["GridSquare"] = Relationship(
+        back_populates="data_collection_group",
+        sa_relationship_kwargs={"cascade": "delete"},
+    )
+    search_maps: List["SearchMap"] = Relationship(
+        back_populates="data_collection_group",
+        sa_relationship_kwargs={"cascade": "delete"},
+    )
 
 
 class NotificationParameter(SQLModel, table=True):  # type: ignore
@@ -468,6 +487,13 @@
     processing_jobs: List["ProcessingJob"] = Relationship(
         back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"}
     )
+    movies: List["Movie"] = Relationship(
+        back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"}
+    )
+    MotionCorrection: List["MotionCorrection"] = Relationship(
+        back_populates="DataCollection"
+    )
+    Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection")
 
 
 class ProcessingJob(SQLModel, table=True):  # type: ignore
@@ -569,6 +595,20 @@
     murfey_ids: List["MurfeyLedger"] = Relationship(
         back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"}
     )
+    MotionCorrection: List["MotionCorrection"] = Relationship(
+        back_populates="DataCollection"
+    )
+    Tomogram: List["Tomogram"] = Relationship(back_populates="AutoProcProgram")
+    CTF: List["CTF"] = Relationship(back_populates="AutoProcProgram")
+    ParticlePicker: List["ParticlePicker"] = Relationship(
+        back_populates="AutoProcProgram"
+    )
+    RelativeIceThickness: List["RelativeIceThickness"] = Relationship(
+        back_populates="AutoProcProgram"
+    )
+    ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship(
+        back_populates="AutoProcProgram"
+    )
 
 
 class MurfeyLedger(SQLModel, table=True):  # type: ignore
@@ -608,6 +648,7 @@
 class GridSquare(SQLModel, table=True):  # type: ignore
     id: Optional[int] = Field(primary_key=True, default=None)
     session_id: int = Field(foreign_key="session.id")
+    atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id")
     name: int
     tag: str
     x_location: Optional[float]
@@ -619,6 +660,13 @@
     thumbnail_size_x: Optional[int]
     thumbnail_size_y: Optional[int]
     pixel_size: Optional[float] = None
+    scaled_pixel_size: Optional[float] = None
+    pixel_location_x: Optional[int] = None
+    pixel_location_y: Optional[int] = None
+    height: Optional[int] = None
+    width: Optional[int] = None
+    angle: Optional[float] = None
+    quality_indicator: Optional[float] = None
     image: str = ""
     session: Optional[Session] = Relationship(back_populates="grid_squares")
     clem_image_series: List["CLEMImageSeries"] = Relationship(
@@ -627,6 +675,9 @@
     foil_holes: List["FoilHole"] = Relationship(
         back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"}
     )
+    data_collection_group: Optional["DataCollectionGroup"] = Relationship(
+        back_populates="grid_squares"
+    )
 
 
 class FoilHole(SQLModel, table=True):  # type: ignore
@@ -643,6 +694,11 @@
     thumbnail_size_x: Optional[int]
     thumbnail_size_y: Optional[int]
     pixel_size: Optional[float] = None
+    scaled_pixel_size: Optional[float] = None
+    pixel_location_x: Optional[int] = None
+    pixel_location_y: Optional[int] = None
+    diameter: Optional[int] = None
+    quality_indicator: Optional[float] = None
     image: str = ""
     grid_square: Optional[GridSquare] = Relationship(back_populates="foil_holes")
     session: Optional[Session] = Relationship(back_populates="foil_holes")
@@ -657,13 +713,21 @@
 class SearchMap(SQLModel, table=True):  # type: ignore
     id: Optional[int] = Field(primary_key=True, default=None)
     session_id: int = Field(foreign_key="session.id")
+    atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id")
     name: str
     tag: str
     x_location: Optional[float] = None
     y_location: Optional[float] = None
     x_stage_position: Optional[float] = None
     y_stage_position: Optional[float] = None
     pixel_size: Optional[float] = None
+    scaled_pixel_size: Optional[float] = None
+    pixel_location_x: Optional[int] = None
+    pixel_location_y: Optional[int] = None
+    scaled_height: Optional[int] = None
+    scaled_width: Optional[int] = None
+    angle: Optional[float] = None
+    quality_indicator: Optional[float] = None
     image: str = ""
     binning: Optional[float] = None
     reference_matrix_m11: Optional[float] = None
@@ -684,17 +748,27 @@
     tilt_series: List["TiltSeries"] = Relationship(
         back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"}
     )
+    data_collection_group: Optional["DataCollectionGroup"] = Relationship(
+        back_populates="search_maps"
+    )
+    Tomogram: List["Tomogram"] = Relationship(back_populates="SearchMap")
 
 
 class Movie(SQLModel, table=True):  # type: ignore
     murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id")
     foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None)
+    data_collection_id: Optional[int] = Field(foreign_key="datacollection.id")
     path: str
     image_number: int
     tag: str
     preprocessed: bool = False
     murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies")
     foil_hole: Optional[FoilHole] = Relationship(back_populates="movies")
+    data_collection: Optional["DataCollection"] = Relationship(back_populates="movies")
+    MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie")
+    TiltImageAlignment: List["TiltImageAlignment"] = Relationship(
+        back_populates="Movie"
+    )
 
 
 class CtfParameters(SQLModel, table=True):  # type: ignore