Merge pull request #419 from LabKey/fb_merge_25.11_to_develop

Merge discvr-25.11 to develop

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/RefNtSequenceModel.java

@@ -18,7 +18,6 @@
 import htsjdk.samtools.util.StringUtil;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
@@ -32,8 +31,11 @@
 import org.labkey.api.exp.api.ExpData;
 import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.files.FileContentService;
+import org.labkey.api.security.Crypt;
 import org.labkey.api.security.User;
+import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.MemTracker;
+import org.labkey.api.util.logging.LogHelper;
 import org.labkey.api.writer.PrintWriters;
 
 import java.io.File;
@@ -55,7 +57,9 @@
  */
 public class RefNtSequenceModel implements Serializable
 {
-    private static final Logger _log = LogManager.getLogger(RefNtSequenceModel.class);
+    private static final Logger _log = LogHelper.getLogger(RefNtSequenceModel.class, "Messages related to Reference NT Sequences");
+
+    public static String BASE_DIRNAME = ".sequences";
 
     private int _rowid;
     private String _name;
@@ -414,7 +418,7 @@ public byte[] getSequenceBases()
 
     public void createFileForSequence(User u, String sequence, @Nullable File outDir) throws IOException
     {
-        File output = getExpectedSequenceFile(outDir);
+        File output = getExpectedSequenceFile();
         if (output.exists())
         {
             output.delete();
@@ -439,9 +443,9 @@ public void createFileForSequence(User u, String sequence, @Nullable File outDir
         Table.update(u, ti, this, _rowid);
     }
 
-    private File getExpectedSequenceFile(@Nullable File outDir) throws IllegalArgumentException
+    public File getExpectedSequenceFile() throws IllegalArgumentException
     {
-        return new File(getSequenceDir(true, outDir), _rowid + ".txt.gz");
+        return FileUtil.appendName(getHashedDir(true), _rowid + ".txt.gz");
     }
 
     private Container getLabKeyContainer()
@@ -455,33 +459,17 @@ private Container getLabKeyContainer()
         return c;
     }
 
-    private File getSequenceDir(boolean create, @Nullable File outDir) throws IllegalArgumentException
+    private File getBaseSequenceDir() throws IllegalArgumentException
     {
         Container c = getLabKeyContainer();
-        File ret = outDir == null ? getReferenceSequenceDir(c) : outDir;
-        if (create && !ret.exists())
-        {
-            ret.mkdirs();
-        }
-
-        return ret;
-    }
-
-    private File getReferenceSequenceDir(Container c) throws IllegalArgumentException
-    {
         FileContentService fileService = FileContentService.get();
         File root = fileService == null ? null : fileService.getFileRoot(c, FileContentService.ContentType.files);
         if (root == null)
         {
             throw new IllegalArgumentException("File root not defined for container: " + c.getPath());
         }
 
-        return new File(root, ".sequences");
-    }
-
-    public void writeSequence(Writer writer, int lineLength) throws IOException
-    {
-        writeSequence(writer, lineLength, null, null);
+        return FileUtil.appendName(root, BASE_DIRNAME);
     }
 
     public void writeSequence(Writer writer, int lineLength, Integer start, Integer end) throws IOException
@@ -562,6 +550,26 @@ public File getOffsetsFile()
             return null;
         }
 
-        return new File(d.getFile().getParentFile(), getRowid() + "_offsets.txt");
+        return FileUtil.appendName(d.getFile().getParentFile(), getRowid() + "_offsets.txt");
+    }
+
+    private File getHashedDir(boolean create)
+    {
+        File baseDir = getBaseSequenceDir();
+        String digest = Crypt.MD5.digest(String.valueOf(getRowid()));
+
+        baseDir = FileUtil.appendName(baseDir, digest.substring(0,4));
+        baseDir = FileUtil.appendName(baseDir, digest.substring(4,8));
+        baseDir = FileUtil.appendName(baseDir, digest.substring(8,12));
+        baseDir = FileUtil.appendName(baseDir, digest.substring(12,20));
+        baseDir = FileUtil.appendName(baseDir, digest.substring(20,28));
+        baseDir = FileUtil.appendName(baseDir, digest.substring(28,32));
+
+        if (create)
+        {
+            baseDir.mkdirs();
+        }
+
+        return baseDir;
     }
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/DockerWrapper.java

@@ -1,5 +1,6 @@
 package org.labkey.api.sequenceanalysis.run;
 
+import org.apache.commons.collections4.list.UnmodifiableList;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
@@ -34,6 +35,7 @@ public class DockerWrapper extends AbstractCommandWrapper
     private boolean _useLocalContainerStorage;
     private String _alternateUserHome = null;
     private final Map<String, String> _dockerEnvironment = new HashMap<>();
+    private int _maxRetries = 3;
 
     public DockerWrapper(String containerName, Logger log, PipelineContext ctx)
     {
@@ -199,7 +201,7 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
 
         localBashScript.setExecutable(true);
         dockerBashScript.setExecutable(true);
-        execute(Arrays.asList("/bin/bash", localBashScript.getPath()));
+        executeWithRetry(Arrays.asList("/bin/bash", localBashScript.getPath()));
 
         if (_useLocalContainerStorage)
         {
@@ -214,6 +216,59 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
         }
     }
 
+    public int getMaxRetries()
+    {
+        return _maxRetries;
+    }
+
+    public void setMaxRetries(int maxRetries)
+    {
+        _maxRetries = maxRetries;
+    }
+
+    // NOTE: when running on a shared/cluster environment with multiple containers initializing concurrently, conflicts can result in these error codes.
+    // As a convenience, build in auto-retry behavior if one of these occurs
+    private final List<Integer> ALLOWABLE_FAIL_CODES = new UnmodifiableList<>(Arrays.asList(125, 127));
+
+    private void executeWithRetry(final List<String> args) throws PipelineJobException
+    {
+        int retries = 0;
+        while (retries <= getMaxRetries())
+        {
+            try
+            {
+                execute(args);
+                break;
+            }
+            catch (PipelineJobException e)
+            {
+                if (ALLOWABLE_FAIL_CODES.contains(getLastReturnCode()))
+                {
+                    retries++;
+                    if (retries > getMaxRetries())
+                    {
+                        getLogger().info("Maximum retries exceeded");
+                        throw e;
+                    }
+
+                    getLogger().info("Exit code " + getLastReturnCode() + ", retrying after 1 sec (" + retries + " of " + getMaxRetries()+ ")");
+                    try
+                    {
+                        Thread.sleep(1000);
+                    }
+                    catch (InterruptedException ex)
+                    {
+                        throw new PipelineJobException(ex);
+                    }
+                }
+                else
+                {
+                    throw e;
+                }
+            }
+        }
+    }
+
     private String getEffectiveContainerName()
     {
         return _containerName;

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage.sql

@@ -1,20 +1,7 @@
-/*
- * Copyright (c) 2012 LabKey Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+PARAMETERS(AnalysisId INTEGER)
+
 select
-  (CAST(a.analysis_id as varchar) || '<>' || a.lineages) as key,
+  (CAST(AnalysisId as varchar) || '<>' || a.lineages) as key,
   a.analysis_id,
   a.lineages,
   max(a.totalLineages) as totalLineages,
@@ -25,13 +12,13 @@ select
   round(100 * (cast(sum(a.total) as float) / cast(max(a.total_reads) as float)), 2) as percent,
   group_concat(distinct a.haplotypesWithAllele) as haplotypesWithAllele,
 
-  CAST((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = a.analysis_id AND s.rowid IN (
-      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.ref_nt_id.locus = a.loci and asj.status = true
+  CAST((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
+      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
     )
   ) as integer) as total_reads_from_locus,
 
-  round(100 * (cast(sum(a.total) as float) / cast((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = a.analysis_id AND s.rowid IN (
-      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.ref_nt_id.locus = a.loci and asj.status = true
+  round(100 * (cast(sum(a.total) as float) / cast((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
+      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
     )
   ) as float)), 2) as percent_from_locus,
   group_concat(distinct a.rowid, ',') as rowids
@@ -47,15 +34,16 @@ FROM (
     group_concat(distinct coalesce(j.ref_nt_id.locus, j.ref_nt_id.name), chr(10)) as loci,
 
     total,
-    cast((select sum(total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = a.analysis_id) as integer) as total_reads,
+    cast((select sum(total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId) as integer) as total_reads,
     group_concat(distinct hs.haplotype, chr(10)) as haplotypesWithAllele
 
   from sequenceanalysis.alignment_summary a
-  join sequenceanalysis.alignment_summary_junction j ON (j.alignment_id = a.rowid and j.status = true)
+  join sequenceanalysis.alignment_summary_junction j ON (j.analysis_id = AnalysisId AND j.alignment_id = a.rowid and j.status = true)
   left join sequenceanalysis.haplotype_sequences hs ON ((
     (hs.name = j.ref_nt_id.lineage AND hs.type = 'Lineage') OR
     (hs.name = j.ref_nt_id.name AND hs.type = 'Allele')
   ) AND hs.haplotype.datedisabled IS NULL)
+  WHERE a.analysis_id = AnalysisId
   group by a.analysis_id, a.rowid, a.total
 
 ) a

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage_pivoted.query.xml

@@ -1,18 +1,5 @@
-/*
- * Copyright (c) 2012 LabKey Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+PARAMETERS(AnalysisId INTEGER)
+
 select
   a.analysis_id,
   a.alleles,
@@ -36,13 +23,13 @@ select
   group_concat(a.rowid, ',') as rowids,
   group_concat(distinct a.haplotypesWithAllele) as haplotypesWithAllele,
 
-  CAST((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = a.analysis_id AND s.rowid IN (
-      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.ref_nt_id.locus = a.loci and asj.status = true
+  CAST((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
+      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
     )
   ) as INTEGER) as total_reads_from_locus,
 
-  round(100 * (cast(sum(a.total) as float) / CASE WHEN count(a.lineages) = 0 THEN max(a.total_reads) ELSE cast((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = a.analysis_id AND s.rowid IN (
-      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.ref_nt_id.locus = a.loci and asj.status = true
+  round(100 * (cast(sum(a.total) as float) / CASE WHEN count(a.lineages) = 0 THEN max(a.total_reads) ELSE cast((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
+      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
     )
   ) as float) END), 2) as percent_from_locus,
   max(lastModified) as lastModified,
@@ -67,14 +54,15 @@ FROM (
     total_forward,
     total_reverse,
     valid_pairs,
-    (select sum(total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = a.analysis_id) as total_reads,
+    (select sum(total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId) as total_reads,
     max(j.modified) as lastModified
   from sequenceanalysis.alignment_summary a
-  left join sequenceanalysis.alignment_summary_junction j ON (j.alignment_id = a.rowid and j.status = true)
+  left join sequenceanalysis.alignment_summary_junction j ON (j.analysis_id = AnalysisId AND j.alignment_id = a.rowid and j.status = true)
   left join sequenceanalysis.haplotype_sequences hs ON ((
     (hs.name = j.ref_nt_id.lineage AND hs.type = 'Lineage') OR
     (hs.name = j.ref_nt_id.name AND hs.type = 'Allele')
   ) AND hs.haplotype.datedisabled IS NULL)
+  WHERE a.analysis_id = AnalysisId
   group by a.analysis_id, a.rowid, a.total, total_forward, total_reverse, valid_pairs
 
 ) a

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage_pivoted.sql

@@ -0,0 +1,5 @@
+-- This is a reversal of SequenceAnalysis-12.329-12.330.sql:
+DROP INDEX IF EXISTS sequenceanalysis.IDX_asj_status_container_alignment_id_ref_nt_id;
+DROP INDEX IF EXISTS sequenceanalysis.IDX_haplotypes_name_date;
+DROP INDEX IF EXISTS sequenceanalysis.IDX_haplotype_sequences_name_haplotype_type;
+DROP INDEX IF EXISTS sequenceanalysis.IDX_alignment_summary_analysis_id_rowid_container_total;

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_grouped.sql

@@ -0,0 +1 @@
+SELECT core.executeJavaUpgradeCode('migrateSequenceDirs');

SequenceAnalysis/resources/schemas/dbscripts/postgresql/SequenceAnalysis-12.330-12.331.sql

@@ -0,0 +1,5 @@
+-- This is a reversal of SequenceAnalysis-12.329-12.330.sql:
+DROP INDEX IDX_asj_status_container_alignment_id_ref_nt_id ON sequenceanalysis.alignment_summary_junction;
+DROP INDEX IDX_haplotypes_name_date ON sequenceanalysis.haplotypes;
+DROP INDEX IDX_haplotype_sequences_name_haplotype_type ON sequenceanalysis.haplotype_sequences;
+DROP INDEX IDX_alignment_summary_analysis_id_rowid_container_total ON sequenceanalysis.alignment_summary;

SequenceAnalysis/resources/schemas/dbscripts/postgresql/SequenceAnalysis-12.331-12.332.sql

@@ -0,0 +1 @@
+EXEC core.executeJavaUpgradeCode 'migrateSequenceDirs';