Skip to content

Commit 73be48a

Browse files
labkey-jeckelslabkey-jeckels
authored
Precompute PTM grouped data for better MAM query perf (#1177)
1 parent 2fbfa12 commit 73be48a

12 files changed

Lines changed: 252 additions & 3 deletions

File tree

‎resources/queries/targetedms/PTMPercentsGrouped.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ SELECT
1818
MAX(ModificationCount) AS ModificationCount @hidden
1919

2020
FROM
21-
PTMPercentsGroupedPrepivot
21+
PTMPercentsGroupedPrepivotCache
2222
GROUP BY
2323
ReplicateName,
2424
Sequence,

‎resources/queries/targetedms/PTMPercentsGroupedPrepivot/.qview.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<customView xmlns="http://labkey.org/data/xml/queryCustomView" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
2+
<sorts>
3+
<sort column="Modification" />
4+
<sort column="Sequence" />
5+
<sort column="ReplicateName" />
6+
</sorts>
7+
</customView>

‎resources/queries/targetedms/PTMPercentsGroupedPrepivotCache.query.xml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<query xmlns="http://labkey.org/data/xml/query">
2+
<metadata>
3+
<tables xmlns="http://labkey.org/data/xml">
4+
<table tableName="PTMPercentsGroupedPrepivotCache" tableDbType="TABLE">
5+
<columns>
6+
<column columnName="PeptideModifiedSequence">
7+
<displayColumnFactory>
8+
<className>org.labkey.targetedms.query.ModifiedSequenceDisplayColumn$PeptideDisplayColumnFactory</className>
9+
<properties>
10+
<property name="showNextAndPrevious">true</property>
11+
<property name="useParens">true</property>
12+
<property name="modificationSite">SiteLocation</property>
13+
</properties>
14+
</displayColumnFactory>
15+
<columnTitle>Sequence</columnTitle>
16+
</column>
17+
<column columnName="TotalPercentModified">
18+
<formatString>0.00%</formatString>
19+
</column>
20+
<column columnName="PercentModified">
21+
<formatString>0.00%</formatString>
22+
</column>
23+
<column columnName="MaxPercentModified">
24+
<formatString>0.00%</formatString>
25+
</column>
26+
<column columnName="Sequence">
27+
<columnTitle>UnmodifiedSequence</columnTitle>
28+
</column>
29+
<column columnName="IsCdr">
30+
<displayColumnFactory>
31+
<className>org.labkey.targetedms.query.CDRDisplayColumnFactory</className>
32+
</displayColumnFactory>
33+
</column>
34+
<column columnName="Risk">
35+
<displayColumnFactory>
36+
<className>org.labkey.targetedms.query.PTMRiskDisplayColumnFactory</className>
37+
</displayColumnFactory>
38+
</column>
39+
</columns>
40+
</table>
41+
</tables>
42+
</metadata>
43+
</query>

‎resources/queries/targetedms/PTMPercentsGroupedPrepivotCache/.qview.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<customView xmlns="http://labkey.org/data/xml/queryCustomView" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
2+
<sorts>
3+
<sort column="Modification" />
4+
<sort column="Sequence" />
5+
<sort column="ReplicateName" />
6+
</sorts>
7+
</customView>

‎resources/schemas/dbscripts/postgresql/targetedms-26.001-26.002.sql‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
CREATE TABLE targetedms.PTMPercentsGroupedPrepivotCache
2+
(
3+
Id BIGINT NOT NULL, -- PeptideId lookup, not a primary key
4+
Container ENTITYID NOT NULL,
5+
RunId BIGINT NOT NULL,
6+
Modification VARCHAR(300) NOT NULL,
7+
TotalPercentModified REAL,
8+
PercentModified REAL,
9+
MaxPercentModified REAL,
10+
ModificationCount INT,
11+
PeptideModifiedSequence VARCHAR(300),
12+
Sequence VARCHAR(300),
13+
PreviousAA VARCHAR(2),
14+
NextAA VARCHAR(2),
15+
SampleFileId BIGINT NOT NULL,
16+
ReplicateName VARCHAR(200),
17+
AminoAcid VARCHAR(5),
18+
SiteLocation VARCHAR(50),
19+
Location INT,
20+
PeptideGroupId BIGINT NOT NULL,
21+
22+
CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_Container FOREIGN KEY (Container) REFERENCES core.Containers(EntityId),
23+
CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_Id FOREIGN KEY (Id) REFERENCES targetedms.Peptide(Id),
24+
CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_RunId FOREIGN KEY (RunId) REFERENCES targetedms.Runs(Id),
25+
CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_SampleFileId FOREIGN KEY (SampleFileId) REFERENCES targetedms.SampleFile(Id),
26+
CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_PeptideGroupId FOREIGN KEY (PeptideGroupId) REFERENCES targetedms.PeptideGroup(Id)
27+
);
28+
29+
CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_Id ON targetedms.PTMPercentsGroupedPrepivotCache(Id);
30+
CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_RunId ON targetedms.PTMPercentsGroupedPrepivotCache(RunId);
31+
CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_Container ON targetedms.PTMPercentsGroupedPrepivotCache(Container);
32+
CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_SampleFileId ON targetedms.PTMPercentsGroupedPrepivotCache(SampleFileId);
33+
CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_PeptideGroupId ON targetedms.PTMPercentsGroupedPrepivotCache(PeptideGroupId);
34+
35+
SELECT core.executeJavaUpgradeCode('populatePTMPercentsGroupedPrepivotCache');

‎resources/schemas/targetedms.xml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1950,4 +1950,26 @@
19501950
<column columnName="SeriesLabel"/>
19511951
</columns>
19521952
</table>
1953+
<table tableName="PTMPercentsGroupedPrepivotCache" tableDbType="TABLE">
1954+
<columns>
1955+
<column columnName="Id"/> <!-- PeptideId lookup, not a primary key -->
1956+
<column columnName="Container"/>
1957+
<column columnName="RunId"/>
1958+
<column columnName="Modification"/>
1959+
<column columnName="TotalPercentModified"/>
1960+
<column columnName="PercentModified"/>
1961+
<column columnName="MaxPercentModified"/>
1962+
<column columnName="ModificationCount"/>
1963+
<column columnName="PeptideModifiedSequence"/>
1964+
<column columnName="Sequence"/>
1965+
<column columnName="PreviousAA"/>
1966+
<column columnName="NextAA"/>
1967+
<column columnName="SampleFileId"/>
1968+
<column columnName="ReplicateName"/>
1969+
<column columnName="AminoAcid"/>
1970+
<column columnName="SiteLocation"/>
1971+
<column columnName="Location"/>
1972+
<column columnName="PeptideGroupId"/>
1973+
</columns>
1974+
</table>
19531975
</tables>

‎src/org/labkey/targetedms/SkylineDocImporter.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,7 @@ else if (folderType == TargetedMSService.FolderType.Library)
514514
parser.logMissingChromatogramCounts();
515515

516516
TargetedMSManager.updateModifiedAreaProportions(_log, run);
517+
TargetedMSManager.populatePTMPercentsGroupedPrepivotCache(run, _user, _container);
517518

518519
if (_pipeRoot.isCloudRoot())
519520
copyExtractedFilesToCloud(run);

‎src/org/labkey/targetedms/TargetedMSManager.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -602,6 +602,11 @@ public static TableInfo getTableInfoQCMetricCache()
602602
return getSchema().getTable(TargetedMSSchema.TABLE_QC_METRIC_CACHE);
603603
}
604604

605+
public static TableInfo getTableInfoPTMPercentsGroupedPrepivotCache()
606+
{
607+
return getSchema().getTable(TargetedMSSchema.TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE);
608+
}
609+
605610
public static TableInfo getTableInfoSkylineAuditLogEntry()
606611
{
607612
return getSchema().getTable(TargetedMSSchema.TABLE_SKYLINE_AUDITLOG_ENTRY);
@@ -1029,6 +1034,12 @@ public static TargetedMSRun[] getRunsInContainer(Container container)
10291034
container.getId(), SkylineDocImporter.STATUS_SUCCESS, Boolean.FALSE);
10301035
}
10311036

1037+
public static TargetedMSRun[] getAllNonDeletedRuns()
1038+
{
1039+
return getRuns("StatusId=? AND deleted=?",
1040+
SkylineDocImporter.STATUS_SUCCESS, Boolean.FALSE);
1041+
}
1042+
10321043
@Nullable
10331044
public static TargetedMSRun getRunByFileName(String fileName, Container container)
10341045
{
@@ -1762,6 +1773,7 @@ private static SQLFragment getTempChromInfoIdsDependentDeleteSql(TableInfo fromT
17621773
/** Actually delete runs that have been marked as deleted from the database */
17631774
private static void purgeDeletedRuns()
17641775
{
1776+
deleteRunDependent(getTableInfoPTMPercentsGroupedPrepivotCache());
17651777
// Delete from FoldChange
17661778
deleteRunDependent(getTableInfoFoldChange());
17671779
// Delete from CalibrationCurve
@@ -2732,6 +2744,63 @@ public static void updateModifiedAreaProportions(@Nullable Logger log, @NotNull
27322744
executor.execute("DROP TABLE " + areasTableName);
27332745
}
27342746

2747+
/**
2748+
* Pre-compute PTMPercentsGroupedPrepivot results during import and store in PTMPercentsCache.
2749+
* Only populates cache for ExperimentMAM folders.
2750+
*/
2751+
public static void populatePTMPercentsGroupedPrepivotCache(@NotNull TargetedMSRun run, @NotNull User user, @NotNull Container container)
2752+
{
2753+
// Delete any existing cache rows for this run
2754+
new SqlExecutor(getSchema()).execute(
2755+
new SQLFragment("DELETE FROM ").append(getTableInfoPTMPercentsGroupedPrepivotCache()).append(" WHERE RunId = ?").add(run.getId()));
2756+
2757+
// Only populate cache for ExperimentMAM folders
2758+
if (getFolderType(container) != TargetedMSService.FolderType.ExperimentMAM)
2759+
{
2760+
return;
2761+
}
2762+
2763+
_log.info("Populating PTMPercentsGroupedPrepivotCache for run " + run.getId());
2764+
2765+
String labkeySql = "SELECT\n" +
2766+
" Modification,\n" +
2767+
" TotalPercentModified,\n" +
2768+
" PercentModified,\n" +
2769+
" MaxPercentModified,\n" +
2770+
" ModificationCount,\n" +
2771+
" Id,\n" +
2772+
" PeptideModifiedSequence,\n" +
2773+
" Sequence,\n" +
2774+
" PreviousAA,\n" +
2775+
" NextAA,\n" +
2776+
" SampleFileId,\n" +
2777+
" ReplicateName,\n" +
2778+
" AminoAcid,\n" +
2779+
" SiteLocation,\n" +
2780+
" Location,\n" +
2781+
" PeptideGroupId\n" +
2782+
"FROM PTMPercentsGroupedPrepivot\n" +
2783+
"WHERE PeptideGroupId.RunId = " + run.getId();
2784+
2785+
UserSchema schema = QueryService.get().getUserSchema(user, container, TargetedMSSchema.SCHEMA_KEY);
2786+
TableInfo tableInfo = QueryService.get().createTable(schema, labkeySql, null, true);
2787+
2788+
SQLFragment insertSql = new SQLFragment();
2789+
insertSql.append("INSERT INTO ").append(getTableInfoPTMPercentsGroupedPrepivotCache());
2790+
insertSql.append(" (Container, RunId, Modification, TotalPercentModified, PercentModified, MaxPercentModified,");
2791+
insertSql.append(" ModificationCount, Id, PeptideModifiedSequence, Sequence,");
2792+
insertSql.append(" PreviousAA, NextAA, SampleFileId, ReplicateName, AminoAcid, SiteLocation, Location, PeptideGroupId)");
2793+
insertSql.append(" SELECT ?, ?, lk.Modification, lk.TotalPercentModified, lk.PercentModified, lk.MaxPercentModified,");
2794+
insertSql.append(" lk.ModificationCount, lk.Id, lk.PeptideModifiedSequence, lk.Sequence,");
2795+
insertSql.append(" lk.PreviousAA, lk.NextAA, lk.SampleFileId, lk.ReplicateName, lk.AminoAcid, lk.SiteLocation, lk.Location, lk.PeptideGroupId");
2796+
insertSql.append(" FROM ").append(tableInfo, "lk");
2797+
insertSql.add(container.getEntityId());
2798+
insertSql.add(run.getId());
2799+
new SqlExecutor(getSchema()).execute(insertSql);
2800+
2801+
_log.info("Finished populating PTMPercentsGroupedPrepivotCache for run " + run.getId());
2802+
}
2803+
27352804
/**
27362805
* Returns the Transition Full Scan settings for the given run. This may be null if the settings were not included
27372806
* in the Skyline document.

‎src/org/labkey/targetedms/TargetedMSModule.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ public String getName()
231231
@Override
232232
public Double getSchemaVersion()
233233
{
234-
return 26.001;
234+
return 26.002;
235235
}
236236

237237
@Override

‎src/org/labkey/targetedms/TargetedMSSchema.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import org.labkey.api.security.permissions.AdminPermission;
7272
import org.labkey.api.targetedms.RepresentativeDataState;
7373
import org.labkey.api.targetedms.RunRepresentativeDataState;
74+
import org.labkey.api.targetedms.TargetedMSService;
7475
import org.labkey.api.util.ContainerContext;
7576
import org.labkey.api.util.HtmlString;
7677
import org.labkey.api.util.Pair;
@@ -232,6 +233,7 @@ public class TargetedMSSchema extends UserSchema
232233
public static final String TABLE_QC_ENABLED_METRICS = "QCEnabledMetrics";
233234
public static final String TABLE_QC_TRACE_METRIC_VALUES = "QCTraceMetricValues";
234235
public static final String TABLE_QC_METRIC_CACHE = "QCMetricCache";
236+
public static final String TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE = "PTMPercentsGroupedPrepivotCache";
235237

236238
public static final String TABLE_GUIDE_SET = "GuideSet";
237239

@@ -1689,6 +1691,14 @@ else if (TABLE_INSTRUMENT_USAGE_PAYMENT.equalsIgnoreCase(name))
16891691
result.getMutableColumnOrThrow("ExperimentRunLSID").setFk(QueryForeignKey.from(_expSchema, cf).to("Runs", "LSID", null));
16901692
}
16911693
TargetedMSTable.fixupLookups(result);
1694+
1695+
if (name.equalsIgnoreCase(TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE))
1696+
{
1697+
// Inject two null columns to match how the uncached version wires up DisplayColumns
1698+
result.addColumn(new ExprColumn(result, "Risk", new SQLFragment("CAST(NULL AS VARCHAR)"), JdbcType.VARCHAR));
1699+
result.addColumn(new ExprColumn(result, "IsCdr", new SQLFragment("CAST(NULL AS VARCHAR)"), JdbcType.VARCHAR));
1700+
}
1701+
16921702
return result;
16931703
}
16941704

@@ -1882,6 +1892,7 @@ private Set<String> getAllTableNames(boolean caseInsensitive)
18821892
hs.add(TABLE_RATE_TYPE);
18831893
hs.add(TABLE_INSTRUMENT_RATE);
18841894
hs.add(TABLE_INSTRUMENT_USAGE_PAYMENT);
1895+
hs.add(TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE);
18851896

18861897
return hs;
18871898
}
@@ -1921,6 +1932,11 @@ private QueryDefinition createRunScopedPTMQuery(String prefix, String baseQueryN
19211932
try
19221933
{
19231934
long runId = Long.parseLong(runIdString);
1935+
if (TargetedMSManager.getFolderType(getContainer()) != TargetedMSService.FolderType.ExperimentMAM)
1936+
{
1937+
throw new IllegalStateException("PTM queries are only supported in ExperimentMAM folders");
1938+
}
1939+
19241940
QueryDefinition queryDef = Objects.requireNonNull(getQueryDef(baseQueryName));
19251941
QueryDefinition result = QueryService.get().createQueryDef(getUser(), getContainer(), getSchemaPath(), queryName);
19261942
result.setSql(queryDef.getSql() + " IN (SELECT r.Name FROM targetedms.Replicate r WHERE r.RunId = " + runId + ")");

0 commit comments

Comments
 (0)