From 98ec4d059cab05b88be1128ee64acbc8febf7cbc Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Fri, 9 Jan 2026 13:33:06 +0000 Subject: [PATCH 1/4] feat(ENGKNOW-2956): Allow not quoted file names for rebases_with_build --- .../main/scala/gorsat/parser/ParseArith.scala | 6 ++++++ .../gorsat/parser/UTestGenomicFunctions.java | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/gortools/src/main/scala/gorsat/parser/ParseArith.scala b/gortools/src/main/scala/gorsat/parser/ParseArith.scala index 7dfdc68e..ee863d0e 100644 --- a/gortools/src/main/scala/gorsat/parser/ParseArith.scala +++ b/gortools/src/main/scala/gorsat/parser/ParseArith.scala @@ -27,6 +27,7 @@ import java.util import gorsat.Analysis.DagMapAnalysis import gorsat.Commands.RowHeader import gorsat.parser.FunctionTypes._ +import gorsat.parser.GenomeFunctions.refBases_with_build import gorsat.parser.ParseUtilities._ import org.gorpipe.exceptions.{GorParsingException, GorSystemException} import org.gorpipe.gor.SyntaxChecker @@ -993,6 +994,11 @@ class ParseArith(rs: GenomicIterator = null) extends JavaTokenParsers with Seria def sfunction: Parser[sFun] = stringIfMatcher | genericFunction[sFun](FunctionTypes.StringFun) | + "REFBASES_WITH_BUILD".ignoreCase ~ "(" ~ sexpr ~ "," ~ iexpr ~ "," ~ iexpr ~ "," ~ myStringLiteralFilename ~ ")" ^^ { + case _ ~ "(" ~ ex1 ~ "," ~ ex2 ~ "," ~ ex3 ~ "," ~ ex4 ~ ")" => (line: ColumnValueProvider) => { + refBases_with_build(this, ex1, ex2, ex3, (line: ColumnValueProvider) => ex4)(line) + } + } | "GTSTAT".ignoreCase ~ "(" ~ sexpr ~ "," ~ iexpr ~ "," ~ sexpr ~ "," ~ sexpr ~ "," ~ iexpr ~ "," ~ sexpr ~ "," ~ sexpr ~ "," ~ iexpr ~ "," ~ sexpr ~ "," ~ sexpr ~ ")" ^^ { case _ ~ "(" ~ ex1 ~ "," ~ ex2 ~ "," ~ ex3 ~ "," ~ ex4 ~ "," ~ ex5 ~ "," ~ ex6 ~ "," ~ ex7 ~ "," ~ ex8 ~ "," ~ ex9 ~ "," ~ ex10 ~ ")" => (line: ColumnValueProvider) => { ParseUtilities.gtStatVCF(ex2(line), ex3(line), ex4(line), ex5(line), ex6(line), ex7(line), ex8(line), ex9(line), ex10(line), refSeq, ex1(line)) diff --git a/gortools/src/test/java/gorsat/parser/UTestGenomicFunctions.java b/gortools/src/test/java/gorsat/parser/UTestGenomicFunctions.java index bf7a0163..6154f6f7 100644 --- a/gortools/src/test/java/gorsat/parser/UTestGenomicFunctions.java +++ b/gortools/src/test/java/gorsat/parser/UTestGenomicFunctions.java @@ -90,16 +90,24 @@ public void testRefBases() { } @Test - public void testRefBasesWithBuild() { + public void testRefBasesWithBuildUnQuoted() { + TestUtils.assertCalculated("refbases_with_build('chr1', 10101, 10103, ../tests/data/ref_mini/chromSeq)", "cct"); + } + + @Test + public void testRefBasesWithBuildUnQuotedField() { + TestUtils.assertCalculated("'noop' | calc a '../tests/data/ref_mini/chromSeq' | calc b refbases_with_build('chr1', 10101, 10103, a)", "cct"); + } + + @Test + public void testRefBasesWithBuildQuoted() { TestUtils.assertCalculated("refbases_with_build('chr1', 10101, 10103, '../tests/data/ref_mini/chromSeq')", "cct"); - // todo test with meaningful input } @Ignore("Read from full build, outside what is in ref_mini, skip as full build is not normally available") @Test public void testRefBasesWithBuild2() { - TestUtils.assertCalculated("refbases_with_build('chr1', 1010101, 1010103, '/private/gorkube-mount/csa/ref/hg19/chromSeq')", "ACC"); - // todo test with meaningful input + TestUtils.assertCalculated("refbases_with_build('chr1', 1010101, 1010103, /private/gorkube-mount/csa/ref/hg19/chromSeq)", "ACC"); } @Test From b1f559eb11c00d38e5c7dee55dd86b7ece1f3c45 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 19 Jan 2026 15:20:22 +0000 Subject: [PATCH 2/4] fix(ENGKNOW-2956): Fix threading issues refbases. --- .../model/gor/iterators/RefSeqFromChromSeq.scala | 11 ++++++++--- .../model/gor/iterators/RefSeqFromConfigFactory.scala | 10 ++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala b/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala index d548a9f6..4c5a3b8b 100644 --- a/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala +++ b/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala @@ -35,9 +35,14 @@ import org.gorpipe.gor.util.DataUtil import org.slf4j.{Logger, LoggerFactory} import java.nio.file.{Files, Path, Paths} +import scala.collection.mutable + +object RefSeqFromConfig { + var downloadTriggered : util.Map[String, java.lang.Boolean] = + new java.util.concurrent.ConcurrentHashMap[String, java.lang.Boolean] +} class RefSeqFromChromSeq(ipath : String, fileReader : FileReader) extends RefSeq { - protected var download_triggered = false private val GOR_REFSEQ_CACHE_FOLDER = System.getProperty("gor.refseq.cache.folder") private val GOR_REFSEQ_CACHE_DOWNLOAD = Option(System.getProperty("gor.refseq.cache.download", "true")).exists(_.toBoolean) @@ -66,8 +71,8 @@ class RefSeqFromChromSeq(ipath : String, fileReader : FileReader) extends RefSeq if (Files.exists(fullCachePath)) { log.debug("Using cached reference build {}", fullCachePath.toString) return fullCachePath.toString - } else if (GOR_REFSEQ_CACHE_DOWNLOAD && !download_triggered) { - download_triggered = true // Only trigger download once per client + } else if (GOR_REFSEQ_CACHE_DOWNLOAD && (RefSeqFromConfig.downloadTriggered.putIfAbsent(fullRefPath.toString, true) == null)) { + // Only trigger download once per client triggerRefSeqDownload(fullRefPath, fullCachePath) } } diff --git a/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromConfigFactory.scala b/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromConfigFactory.scala index 600e2a8b..d34c5412 100644 --- a/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromConfigFactory.scala +++ b/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromConfigFactory.scala @@ -24,19 +24,13 @@ package org.gorpipe.model.gor.iterators import org.gorpipe.gor.model.FileReader -import java.util -import java.util.Collections - class RefSeqFromConfigFactory(path: String, reader: FileReader) extends RefSeqFactory { - var refSeqMap = Collections.synchronizedMap(new util.HashMap[String, RefSeq]()) override def create(): RefSeq = { - refSeqMap.computeIfAbsent(path, _ => { - new RefSeqFromChromSeq(path, reader)}) + new RefSeqFromChromSeq(path, reader) } def create(altPath: String): RefSeq = { - refSeqMap.computeIfAbsent(altPath, _ => { - new RefSeqFromChromSeq(altPath, reader)}) + new RefSeqFromChromSeq(altPath, reader) } } From 039759048f732bf91cd88a760171774fd14cd7b7 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 19 Jan 2026 18:36:19 +0000 Subject: [PATCH 3/4] fix(ENGKNOW-2956): Fix threading issues refbases. --- .../gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala | 7 +++---- ...tRefSeqFromConfig.java => UTestRefSeqFromChromSeq.java} | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) rename model/src/test/java/org/gorpipe/model/gor/iterators/{UTestRefSeqFromConfig.java => UTestRefSeqFromChromSeq.java} (98%) diff --git a/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala b/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala index 4c5a3b8b..bf7158f9 100644 --- a/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala +++ b/model/src/main/scala/org/gorpipe/model/gor/iterators/RefSeqFromChromSeq.scala @@ -35,10 +35,9 @@ import org.gorpipe.gor.util.DataUtil import org.slf4j.{Logger, LoggerFactory} import java.nio.file.{Files, Path, Paths} -import scala.collection.mutable -object RefSeqFromConfig { - var downloadTriggered : util.Map[String, java.lang.Boolean] = +object RefSeqFromChromSeq { + var downloadTriggered : util.Map[String, java.lang.Boolean] = new java.util.concurrent.ConcurrentHashMap[String, java.lang.Boolean] } @@ -71,7 +70,7 @@ class RefSeqFromChromSeq(ipath : String, fileReader : FileReader) extends RefSeq if (Files.exists(fullCachePath)) { log.debug("Using cached reference build {}", fullCachePath.toString) return fullCachePath.toString - } else if (GOR_REFSEQ_CACHE_DOWNLOAD && (RefSeqFromConfig.downloadTriggered.putIfAbsent(fullRefPath.toString, true) == null)) { + } else if (GOR_REFSEQ_CACHE_DOWNLOAD && (RefSeqFromChromSeq.downloadTriggered.putIfAbsent(fullRefPath.toString, true) == null)) { // Only trigger download once per client triggerRefSeqDownload(fullRefPath, fullCachePath) } diff --git a/model/src/test/java/org/gorpipe/model/gor/iterators/UTestRefSeqFromConfig.java b/model/src/test/java/org/gorpipe/model/gor/iterators/UTestRefSeqFromChromSeq.java similarity index 98% rename from model/src/test/java/org/gorpipe/model/gor/iterators/UTestRefSeqFromConfig.java rename to model/src/test/java/org/gorpipe/model/gor/iterators/UTestRefSeqFromChromSeq.java index 35b783b4..8f094845 100644 --- a/model/src/test/java/org/gorpipe/model/gor/iterators/UTestRefSeqFromConfig.java +++ b/model/src/test/java/org/gorpipe/model/gor/iterators/UTestRefSeqFromChromSeq.java @@ -13,7 +13,7 @@ import java.nio.file.Files; import java.nio.file.Path; -public class UTestRefSeqFromConfig { +public class UTestRefSeqFromChromSeq { @Rule public final ProvideSystemProperty cacheFolder @@ -121,8 +121,7 @@ public void testGetRefbaseFromCache() throws InterruptedException { RefSeqFromChromSeq refseq = new RefSeqFromChromSeq("../tests/data/ref_mini/chromSeq", new DriverBackedFileReader("")); - refseq.download_triggered_$eq(false); - + RefSeqFromChromSeq.downloadTriggered().clear(); Assert.assertEquals('C', refseq.getBase("chr1", 101000)); From 7435968988baa2343d75b542e2b5ff3390837c2e Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 19 Jan 2026 19:12:02 +0000 Subject: [PATCH 4/4] fix(ENGKNOW-2956): Fix threading issues refbases. --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 11d9efa3..1802e771 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.8.0 +5.8.1