Skip to content

[bot] Merge 25.7 to 25.11 #1531

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
github-actions merged 6 commits into from
Nov 4, 2025
Merged

[bot] Merge 25.7 to 25.11 #1531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e4c0f16
Cage audit log process ETL (#1520)
kollil Oct 30, 2025
b9b0f62
Add row count validation to ETL (#1527)
bbimber Oct 30, 2025
1429e0e
Fix per diems when dual-assigned research projects (#1511)
brentlogan Oct 30, 2025
01af55c
Modified assignment project process so that "projected Release date" …
Ohsudev Nov 3, 2025
fab2c8d
25.7 fb parentage evaluation (#1526)
Ohsudev Nov 3, 2025
c593d7c
Merge 25.7 to 25.11
labkey-teamcity Nov 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions GeneticsCore/resources/etls/MHC_Typing.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,21 @@
</alternateKeys>
</destination>
</transform>

<transform id="rc" type="TaskrefTransformStep">
<taskref ref="org.labkey.primeseq.etl.VerifyRowCount">
<settings>
<setting name="sourceRemoteSource" value="PRIMESEQ_MHC"/>
<setting name="sourceSchema" value="geneticscore"/>
<setting name="sourceQuery" value="mhc_data"/>
<setting name="sourceColumn" value="objectId"/>

<setting name="destSchema" value="geneticscore"/>
<setting name="destQuery" value="mhc_data"/>
<setting name="destColumn" value="objectId"/>
</settings>
</taskref>
</transform>
</transforms>

<incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified">
Expand Down
365 changes: 180 additions & 185 deletions onprc_billing/resources/queries/onprc_billing/perDiemsByDay.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,193 +2,188 @@
SELECT
t.*,
CASE
WHEN t.overlappingProjects IS NULL then 1
WHEN t.tmbAssignments > 0 then 0 --note: when co-assigned to TMB, per diem is never charged. if TMB is single-assigned, it needs to pay per diem, and will be caught above
WHEN t.overlappingProjects IS NULL THEN 1
-- NOTE: An assignment overlapping with TMB is not charged per diems. If TMB is single-assigned, it pays per diem and will be caught above.
WHEN t.tmbAssignments > 0 then 0
WHEN t.assignedProject IS NULL AND t.overlappingProjects IS NOT NULL THEN 0
WHEN t.ProjectType != 'Research' and t.overlappingProjectsCategory LIKE '%Research%' Then 0
WHEN t.ProjectType != 'Research' and t.overlappingProjectsCategory NOT LIKE '%Research%' Then (1.0 / NULLIF((t.totalOverlappingProjects + 1), 0))
WHEN t.ProjectType = 'Research' and t.overlappingProjectsCategory NOT LIKE '%Research%' Then 1
WHEN t.ProjectType = 'Research' and t.overlappingProjectsCategory LIKE '%Research%' Then (1.0 / NULLIF((t.totalOverlappingResearchProjects + 1), 0))
ELSE 1
END as effectiveDays,
CASE
WHEN (t.assignedProject IS NULL AND t.overlappingProjects IS NULL) THEN 'Base Grant'
WHEN t.overlappingProjects IS NULL then 'Single Project'
WHEN (t.tmbAssignments > 0) then 'Exempt By TMB'
WHEN (t.isTMBProject = 1 AND t.overlappingProjects IS NOT NULL) then 'Exempt By TMB'
WHEN t.assignedProject IS NULL AND t.overlappingProjects IS NOT NULL THEN 'Paid By Overlapping Project'
WHEN t.ProjectType != 'Research' and t.overlappingProjectsCategory LIKE '%Research%' Then 'Paid By Overlapping Project'
WHEN t.ProjectType != 'Research' and t.overlappingProjectsCategory NOT LIKE '%Research%' Then 'Multiple Resources'
WHEN t.ProjectType = 'Research' and t.overlappingProjectsCategory NOT LIKE '%Research%' Then 'Single Project'
WHEN t.ProjectType = 'Research' and t.overlappingProjectsCategory LIKE '%Research%' Then 'Multiple Research'
ELSE 'Unknown'
END as category,

CASE
WHEN (t.perDiemFeeCount > 1) THEN null --this should catch duplicate chargeIds
--use the treatmentOtrder to look for BottleFed
WHEN (t.bottleFedRecordCount > 0 AND t.researchRecordCount > 0) THEN maxPdfChargeId
--if this item supports infants, charge that

WHEN (pdfChargeInfantCount > 0 AND maxPdfChargeId IS NOT NULL) THEN maxPdfChargeId
--otherwise infants are a special rate
WHEN (perDiemAge < CAST(javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.INFANT_PER_DIEM_AGE') AS INTEGER)) THEN (SELECT ci.rowid FROM onprc_billing_public.chargeableItems ci WHERE ci.name = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.INFANT_PER_DIEM'))
--add quarantine flags, which trump housing type
WHEN (quarantineFlagCount > 0) THEN (SELECT ci.rowid FROM onprc_billing_public.chargeableItems ci WHERE ci.name = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.QUARANTINE_PER_DIEM'))
--finally defer to housing condition
ELSE maxPdfChargeId
END as chargeId,

--find overlapping tier flags on that day
coalesce((
SELECT group_concat(DISTINCT f.flag.value) as tier
FROM study.flags f
--NOTE: allow flags that ended on this date
WHERE f.Id = t.Id AND f.enddateCoalesced >= t.dateOnly AND f.dateOnly <= t.dateOnly AND f.flag.category = 'Housing Tier'
), 'Tier 2') as tier

FROM (

SELECT
i2.Id,
CAST(CAST(i2.dateOnly as date) as timestamp) as date,
i2.dateOnly @hidden,
coalesce(a.project, (SELECT p.project FROM ehr.project p WHERE p.name = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.BASE_GRANT_PROJECT'))) as project,
a.project as assignedProject,
max(a.duration) as duration, --should only have 1 value, no so need to include in grouping
max(timestampdiff('SQL_TSI_DAY', d.birth, i2.dateOnly)) as ageAtTime,
a.project.use_Category as ProjectType,
count(*) as totalAssignmentRecords,
group_concat(DISTINCT a2.project.displayName) as overlappingProjects,
count(DISTINCT a2.project) as totalOverlappingProjects,
sum(CASE WHEN a2.project.use_Category = 'Research' THEN 1 ELSE 0 END) as totalOverlappingResearchProjects,
group_concat(DISTINCT a2.project.use_category) as overlappingProjectsCategory,
group_concat(DISTINCT a2.project.protocol) as overlappingProtocols,
count(h3.room) as totalHousingRecords,
group_concat(DISTINCT h3.room) as rooms,
group_concat(DISTINCT h3.cage) as cages,
group_concat(DISTINCT h3.objectid) as housingRecords,
group_concat(DISTINCT a.objectid) as assignmentRecords,
group_concat(DISTINCT h3.room.housingCondition.value) as housingConditions,
group_concat(DISTINCT h3.room.housingType.value) as housingTypes,
max(timestampdiff('SQL_TSI_DAY', d.birth, i2.dateOnly)) as perDiemAge,
count(DISTINCT pdf.chargeId) as perDiemFeeCount,
count(t1.code) AS bottleFedRecordCount,
count(a3.project) AS researchRecordCount,
count(CASE WHEN pdf.canChargeInfants = true THEN 1 ELSE null END) AS pdfChargeInfantCount,
max(pdf.chargeId) AS maxPdfChargeId,
(SELECT count(*) AS c FROM study.flags q WHERE q.Id = i2.Id AND q.flag.value LIKE '%Quarantine%' AND q.dateOnly <= i2.dateOnly AND q.enddateCoalesced >= i2.dateOnly) AS quarantineFlagCount,
max(i2.startDate) as startDate @hidden,
count(tmb.Id) as tmbAssignments,
SUM(CASE WHEN a.projectName = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.TMB_PROJECT') THEN 1 ELSE 0 END) as isTMBProject
WHEN t.ProjectType != 'Research' AND t.overlappingProjectsCategory LIKE '%Research%' THEN 0
WHEN t.ProjectType != 'Research' AND t.overlappingProjectsCategory NOT LIKE '%Research%' THEN (1.0 / NULLIF((t.totalOverlappingProjects + 1), 0))
WHEN t.ProjectType = 'Research' AND t.overlappingProjectsCategory NOT LIKE '%Research%' THEN 1
WHEN t.ProjectType = 'Research' AND t.overlappingProjectsCategory LIKE '%Research%' THEN (1.0 / NULLIF((t.totalOverlappingResearchProjects + 1), 0))
ELSE 1
END as effectiveDays,
CASE
WHEN (t.assignedProject IS NULL AND t.overlappingProjects IS NULL) THEN 'Base Grant'
WHEN t.overlappingProjects IS NULL then 'Single Project'
WHEN (t.tmbAssignments > 0) THEN 'Exempt By TMB'
WHEN (t.isTMBProject = 1 AND t.overlappingProjects IS NOT NULL) THEN 'Exempt By TMB'
WHEN t.assignedProject IS NULL AND t.overlappingProjects IS NOT NULL THEN 'Paid By Overlapping Project'
WHEN t.ProjectType != 'Research' AND t.overlappingProjectsCategory LIKE '%Research%' THEN 'Paid By Overlapping Project'
WHEN t.ProjectType != 'Research' AND t.overlappingProjectsCategory NOT LIKE '%Research%' THEN 'Multiple Resources'
WHEN t.ProjectType = 'Research' AND t.overlappingProjectsCategory NOT LIKE '%Research%' THEN 'Single Project'
WHEN t.ProjectType = 'Research' AND t.overlappingProjectsCategory LIKE '%Research%' THEN 'Multiple Research'
ELSE 'Unknown'
END as category,
CASE
-- Catch duplicate chargeIds
WHEN (t.perDiemFeeCount > 1) THEN NULL
-- Use the treatmentOrder to look for BottleFed
WHEN (t.bottleFedRecordCount > 0 AND t.researchRecordCount > 0) THEN maxPdfChargeId
-- If this item supports infants, charge that
WHEN (pdfChargeInfantCount > 0 AND maxPdfChargeId IS NOT NULL) THEN maxPdfChargeId
-- Otherwise, infants are a special rate
WHEN (perDiemAge < CAST(javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.INFANT_PER_DIEM_AGE') AS INTEGER))
THEN (SELECT ci.rowid FROM onprc_billing_public.chargeableItems ci WHERE ci.name = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.INFANT_PER_DIEM'))
-- Add quarantine flags, which trump housing type
WHEN (quarantineFlagCount > 0)
THEN (SELECT ci.rowid FROM onprc_billing_public.chargeableItems ci WHERE ci.name = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.QUARANTINE_PER_DIEM'))
-- Finally, defer to housing condition
ELSE maxPdfChargeId
END as chargeId,
-- Find overlapping tier flags on that day
coalesce((
SELECT group_concat(DISTINCT f.flag.value) AS tier
FROM study.flags f
-- NOTE: allow flags that ended on this date
WHERE f.Id = t.Id AND f.enddateCoalesced >= t.dateOnly AND f.dateOnly <= t.dateOnly AND f.flag.category = 'Housing Tier'
), 'Tier 2') AS tier

FROM (
-- find all distinct animals housed here on each day. this was moved to be the
-- first join so we can be sure to include any animal housed here on that day,
-- as opposed to only assigned animals
SELECT
h.Id,
i.dateOnly,
max(h.date) as lastHousingStart,
min(i.startDate) as startDate @hidden
FROM ldk.dateRange i
JOIN study.housing h ON (h.dateOnly <= i.dateOnly AND h.enddateCoalesced >= i.dateOnly AND h.qcstate.publicdata = true)
--WHERE i.dateOnly <= curdate()
GROUP BY h.Id, i.dateOnly
) i2

JOIN study.demographics d ON (
i2.Id = d.Id
)

LEFT JOIN study.treatment_Order t1 ON t1.id = i2.id AND t1.code.meaning LIKE '%Bottle%' AND t1.date <=i2.dateOnly

LEFT JOIN study.assignment a3 ON a3.id = i2.id AND a3.date <= i2.dateOnly AND a3.endDateCoalesced > i2.dateOnly AND a3.project.Use_Category LIKE '%Research%'

-- housing is a little tricky. using the query above, we want to find the max start date, on or before this day
-- the housingType from this room will be used
JOIN study.housing h3 ON (h3.Id = i2.Id AND i2.lastHousingStart = h3.date AND h3.qcstate.publicdata = true)

--then join to any assignment record overlapping each day
LEFT JOIN (
SELECT
a.lsid,
a.id,
a.project,
a.project.name as projectName,
a.date,
a.assignCondition,
a.releaseCondition,
a.projectedReleaseCondition,
a.duration,
a.enddate,
a.dateOnly,
a.enddateCoalesced,
a.objectid
FROM study.assignment a

WHERE a.qcstate.publicdata = true
--NOTE: we might want to exclude 1-day assignments, or deal with them differently
--AND a.duration > 0

) a ON (
i2.Id = a.id AND
a.dateOnly <= i2.dateOnly
--assignments end at midnight, so an assignment doesnt count on the current date if it ends on it
--NOTE: we do need to capture 1-day assignments, so these do count if the start and end are the same day
AND (a.enddate IS NULL OR a.enddateCoalesced > i2.dateOnly OR (a.dateOnly = i2.dateOnly AND a.enddateCoalesced = i2.dateOnly))
)

LEFT JOIN (
--for each assignment, find co-assigned projects on that day
SELECT
a2.lsid,
a2.date,
a2.enddate,
a2.id,
a2.project,
a2.dateOnly,
a2.enddateCoalesced
FROM study.assignment a2
WHERE
--NOTE: this has been reversed. if one-day assignments are exempted from per diems, this should be restored
--exclude 1-day assignments
--a2.duration > 1 AND
a2.qcstate.publicdata = true
) a2 ON (
i2.id = a2.id
AND a2.dateOnly <= i2.dateOnly
AND a.project != a2.project
--assignments end at midnight, so an assignment doesnt count on the current date if it ends on it
--we also need to include 1-day assignments
AND (a2.enddate IS NULL OR a2.enddateCoalesced > i2.dateOnly OR (a2.dateOnly = i2.dateOnly AND a2.enddateCoalesced = i2.dateOnly))
AND a.lsid != a2.lsid
)

--find overlapping TMB on this date, which overrides the per diem
--note: also include them if the current project is TMB, as this is also exempt
LEFT JOIN study.assignment tmb ON (
a.id = tmb.id
AND tmb.dateOnly <= i2.dateOnly
and tmb.project != a.project
AND tmb.endDateCoalesced >= i2.dateOnly
AND tmb.project.name = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.TMB_PROJECT')
)

LEFT JOIN onprc_billing.perDiemFeeDefinition pdf
ON (
pdf.housingType = h3.room.housingType AND
pdf.housingDefinition = h3.room.housingCondition AND

--find overlapping tier flags on that day
coalesce((
SELECT group_concat(DISTINCT f.flag.value) as tier
FROM study.flags f
--NOTE: allow flags that ended on this date
WHERE f.Id = i2.Id AND f.enddateCoalesced >= i2.dateOnly AND f.dateOnly <= i2.dateOnly AND f.flag.category = 'Housing Tier'
), 'Tier 2') = pdf.tier
)

GROUP BY i2.dateOnly, I2.Id, a.project, a.project.use_Category
SELECT
i2.Id,
CAST(CAST(i2.dateOnly AS DATE) AS TIMESTAMP) AS DATE,
i2.dateOnly @hidden,
coalesce(a.project, (SELECT p.project FROM ehr.project p WHERE p.name = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.BASE_GRANT_PROJECT'))) AS project,
a.project AS assignedProject,
max(a.duration) AS duration, -- should only have 1 value, no so need to include in grouping
max(timestampdiff('SQL_TSI_DAY', d.birth, i2.dateOnly)) AS ageAtTime,
a.project.use_Category AS ProjectType,
count(*) AS totalAssignmentRecords,
group_concat(DISTINCT a2.project.displayName) AS overlappingProjects,
count(DISTINCT a2.project) AS totalOverlappingProjects,
sum(CASE WHEN a2.project.use_Category = 'Research' THEN 1 ELSE 0 END) as totalOverlappingResearchProjects,
group_concat(DISTINCT a2.project.use_category) AS overlappingProjectsCategory,
group_concat(DISTINCT a2.project.protocol) AS overlappingProtocols,
count(h3.room) AS totalHousingRecords,
group_concat(DISTINCT h3.room) AS rooms,
group_concat(DISTINCT h3.cage) AS cages,
group_concat(DISTINCT h3.objectid) AS housingRecords,
group_concat(DISTINCT a.objectid) AS assignmentRecords,
group_concat(DISTINCT h3.room.housingCondition.value) AS housingConditions,
group_concat(DISTINCT h3.room.housingType.value) AS housingTypes,
max(timestampdiff('SQL_TSI_DAY', d.birth, i2.dateOnly)) AS perDiemAge,
count(DISTINCT pdf.chargeId) AS perDiemFeeCount,
i2.researchRecordCount,
i2.bottleFedRecordCount,
count(CASE WHEN pdf.canChargeInfants = TRUE THEN 1 ELSE NULL END) AS pdfChargeInfantCount,
max(pdf.chargeId) AS maxPdfChargeId,
(SELECT count(*) AS c
FROM study.flags q
WHERE q.Id = i2.Id AND q.flag.value LIKE '%Quarantine%' AND q.dateOnly <= i2.dateOnly AND q.enddateCoalesced >= i2.dateOnly
) AS quarantineFlagCount,
max(i2.startDate) AS startDate @hidden,
count(tmb.Id) AS tmbAssignments,
SUM(CASE WHEN a.projectName = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.TMB_PROJECT') THEN 1 ELSE 0 END) AS isTMBProject

FROM (
-- Find all distinct animals housed at the Center each day.
-- This is the first dataset to include all animals here, not just assigned animals.
SELECT
h.Id,
i.dateOnly,
max(h.date) AS lastHousingStart,
min(i.startDate) AS startDate @hidden,
count(a3.project) as researchRecordCount,
count(t1.code) as bottleFedRecordCount
FROM ldk.dateRange i
JOIN study.housing h ON (h.dateOnly <= i.dateOnly AND h.enddateCoalesced >= i.dateOnly AND h.qcstate.publicdata = TRUE)
LEFT JOIN study.assignment a3 ON a3.id = h.id AND a3.date <= i.dateOnly AND a3.endDateCoalesced > i.dateOnly AND a3.project.Use_Category LIKE '%Research%'
LEFT JOIN study.treatment_Order t1 ON t1.id = h.id AND t1.code.meaning LIKE '%Bottle%' AND t1.date <= i.dateOnly
GROUP BY h.Id, i.dateOnly
) i2

JOIN study.demographics d ON (
i2.Id = d.Id
)

-- Housing is a little tricky. Using the query above, we want to find the max start date, on or before this day.
-- The housingType from this location is used.
JOIN study.housing h3 ON (h3.Id = i2.Id AND i2.lastHousingStart = h3.date AND h3.qcstate.publicdata = TRUE)

-- Then join to any assignment record overlapping each day
LEFT JOIN (
SELECT
a.lsid,
a.id,
a.project,
a.project.name AS projectName,
a.date,
a.assignCondition,
a.releaseCondition,
a.projectedReleaseCondition,
a.duration,
a.enddate,
a.dateOnly,
a.enddateCoalesced,
a.objectid
FROM study.assignment a
WHERE a.qcstate.publicdata = TRUE
-- NOTE: We don't exclude 1-day assignments or treat them differently.
-- AND a.duration > 0
) a ON (
i2.Id = a.id
AND a.dateOnly <= i2.dateOnly
-- Assignments end at midnight, so an assignment doesn't count on the current date if it ends on it.
-- However, we also include 1-day assignments, which *can* have the end date match the start date.
AND (a.enddate IS NULL OR a.enddateCoalesced > i2.dateOnly OR (a.dateOnly = i2.dateOnly AND a.enddateCoalesced = i2.dateOnly))
)

LEFT JOIN (
-- For each assignment, find the co-assigned projects on that day.
SELECT
a2.lsid,
a2.date,
a2.enddate,
a2.id,
a2.project,
a2.dateOnly,
a2.enddateCoalesced
FROM study.assignment a2
WHERE a2.qcstate.publicdata = TRUE
-- NOTE: We don't exclude 1-day assignments or treat them differently.
-- AND a2.duration > 1
) a2 ON (
i2.id = a2.id
AND a2.dateOnly <= i2.dateOnly
AND a.project != a2.project
-- Assignments end at midnight, so an assignment doesn't count on the current date if it ends on it.
-- However, we also include 1-day assignments, which *can* have the end date match the start date.
AND (a2.enddate IS NULL OR a2.enddateCoalesced > i2.dateOnly OR (a2.dateOnly = i2.dateOnly AND a2.enddateCoalesced = i2.dateOnly))
AND a.lsid != a2.lsid
)

-- Find overlapping TMB on this date, which overrides the per diem.
LEFT JOIN study.assignment tmb ON (
a.id = tmb.id
AND tmb.dateOnly <= i2.dateOnly
AND tmb.project != a.project
AND tmb.endDateCoalesced >= i2.dateOnly
AND tmb.project.name = javaConstant('org.labkey.onprc_ehr.ONPRC_EHRManager.TMB_PROJECT')
)

LEFT JOIN onprc_billing.perDiemFeeDefinition pdf ON (
pdf.housingType = h3.room.housingType
AND pdf.housingDefinition = h3.room.housingCondition
-- Find overlapping tier flags on that day
AND coalesce(
(SELECT group_concat(DISTINCT f.flag.value) as tier
FROM study.flags f
--NOTE: allow flags that ended on this date
WHERE f.Id = i2.Id AND f.enddateCoalesced >= i2.dateOnly AND f.dateOnly <= i2.dateOnly AND f.flag.category = 'Housing Tier'),
'Tier 2'
) = pdf.tier
)

GROUP BY i2.dateOnly, I2.Id, a.project, a.project.use_Category, i2.researchRecordCount, i2.bottleFedRecordCount

) t
Loading