Skip to content

Commit 10cc88b

Browse files
gmagnugmagnu
authored
fix(ENGKNOW-2781): Change link defaults and fix some errors (#94)
* fat(ENGKNOW-2781): Minor tweaks to the MDR error handling. * fat(ENGKNOW-2781): Fix minor issue with Exec. * fat(ENGKNOW-2781): Changes to make -link an pgor/paralle work correctly. * fat(ENGKNOW-2781): Minor refactoring. Cache temp file name for FileTable. * feat(ENGKNOW-2781): Update dict link file support. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Fixes - Fix handling of thedict.gord.link - Fix link validation issue. * fix(ENGKNOW-2781): Link file defaults changed. Links default to V0 unless -link option is used. * fix(ENGKNOW-2781): Link file defaults changed. Links default to V0 unless -link option is used.
1 parent 501e346 commit 10cc88b

10 files changed

Lines changed: 112 additions & 55 deletions

File tree

‎drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java‎

Lines changed: 65 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ public void testProjectRead() throws IOException {
128128
Assert.assertEquals("S3", source.getSourceType().getName());
129129
Assert.assertEquals("s3://gdb-unit-test-data/projects/some_project/BVL_FATHER_SLC52A2/BVL_FATHER_SLC52A2.vcf.gz.gorz", source.getFullPath());
130130

131-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
131+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
132132
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHUUUART", iterator.getHeader());
133133
}
134134
}
@@ -144,7 +144,7 @@ public void testProjectReadUserData() throws IOException {
144144
Assert.assertEquals("S3", source.getSourceType().getName());
145145
Assert.assertEquals("s3://gdb-unit-test-data/projects/some_project/user_data/BVL_INDEX_SLC52A2/BVL_INDEX_SLC52A2.vcf.gz.gorz", source.getFullPath());
146146

147-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
147+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
148148
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNVKUKNN", iterator.getHeader());
149149
}
150150
}
@@ -161,7 +161,7 @@ public void testProjectSharedRead() throws IOException {
161161
Assert.assertEquals("S3", source.getSourceType().getName());
162162
Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_MOTHER_SLC52A2/BVL_MOTHER_SLC52A2.vcf.gz.gorz", source.getFullPath());
163163

164-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
164+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
165165
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNKHARLQ", iterator.getHeader());
166166
}
167167
}
@@ -177,7 +177,7 @@ public void testRegionSharedRead() throws IOException {
177177
Assert.assertEquals("S3", source.getSourceType().getName());
178178
Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_MOTHER_SLC52A2/BVL_MOTHER_SLC52A2.vcf.gz.gorz", source.getFullPath());
179179

180-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
180+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
181181
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNKHARLQ", iterator.getHeader());
182182
}
183183
}
@@ -193,7 +193,7 @@ public void testGlobalSharedRead() throws IOException {
193193
Assert.assertEquals("S3", source.getSourceType().getName());
194194
Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_MOTHER_SLC52A2/BVL_MOTHER_SLC52A2.vcf.gz.gorz", source.getFullPath());
195195

196-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
196+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
197197
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNKHARLQ", iterator.getHeader());
198198
}
199199
}
@@ -209,7 +209,7 @@ public void testProjectFileRead() throws IOException {
209209
Assert.assertEquals("S3", source.getSourceType().getName());
210210
Assert.assertEquals("s3://gdb-unit-test-data/projects/some_project/BVL_MOTHER_SLC52A2.vcf.gz.gorz", source.getFullPath());
211211

212-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
212+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
213213
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNKHARLQ", iterator.getHeader());
214214
}
215215
}
@@ -225,7 +225,7 @@ public void testProjectSharedFileRead() throws IOException {
225225
Assert.assertEquals("S3", source.getSourceType().getName());
226226
Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_FATHER_SLC52A2.vcf.gz.gorz", source.getFullPath());
227227

228-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
228+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
229229
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHUUUART", iterator.getHeader());
230230
}
231231
}
@@ -241,7 +241,7 @@ public void testRegionSharedFileRead() throws IOException {
241241
Assert.assertEquals("S3", source.getSourceType().getName());
242242
Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_FATHER_SLC52A2.vcf.gz.gorz", source.getFullPath());
243243

244-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
244+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
245245
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHUUUART", iterator.getHeader());
246246
}
247247
}
@@ -257,14 +257,14 @@ public void testGlobalSharedFileRead() throws IOException {
257257
Assert.assertEquals("S3", source.getSourceType().getName());
258258
Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_FATHER_SLC52A2.vcf.gz.gorz", source.getFullPath());
259259

260-
try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
260+
try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) {
261261
Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHUUUART", iterator.getHeader());
262262
}
263263
}
264264

265265
@Test
266266
public void testReadWithLinkFile() throws IOException {
267-
Path gorRoot = workDirPath.resolve("some_project");
267+
Path gorRoot = workDirPath.resolve("some_project");
268268
Path linkFile = gorRoot.resolve(DataUtil.toLinkFile("a", DataType.GORZ));
269269
Files.createDirectory(gorRoot);
270270
Files.writeString(linkFile, DataUtil.toFile(DataUtil.toFile("s3data://project/user_data/BVL_INDEX_SLC52A2", DataType.VCFGZ), DataType.GORZ));
@@ -290,7 +290,7 @@ public void testReadWithLinkFile() throws IOException {
290290

291291
@Test
292292
public void testReadWithUnaccessableLinkFile() throws IOException {
293-
Path gorRoot = workDirPath.resolve("some_project");
293+
Path gorRoot = workDirPath.resolve("some_project");
294294
Path linkFile = workDirPath.resolve(DataUtil.toLinkFile("a", DataType.GORZ));
295295
Files.createDirectory(gorRoot);
296296
Files.writeString(linkFile, DataUtil.toFile(DataUtil.toFile("s3data://project/user_data/BVL_INDEX_SLC52A2", DataType.VCFGZ), DataType.GORZ));
@@ -303,12 +303,12 @@ public void testReadWithUnaccessableLinkFile() throws IOException {
303303

304304
var e2 = Assert.assertThrows("Should not be resolved, link outside project", GorSecurityException.class, () -> fileReader.resolveUrl(DataUtil.toLinkFile("../a", DataType.GORZ)));
305305
Assert.assertTrue(e2.getMessage().contains("File paths must be within project scope"));
306-
}
306+
}
307307

308308
@Test
309309
@Ignore("Slow test, meant to be manually run")
310310
public void testReadServer() throws IOException {
311-
Path gorRoot = workDirPath.resolve("some_project");
311+
Path gorRoot = workDirPath.resolve("some_project");
312312
Path linkFile = gorRoot.resolve(DataUtil.toLinkFile("a", DataType.GORZ));
313313
Files.createDirectory(gorRoot);
314314
//Files.write(linkFile, "s3data://project/user_data/BVL_INDEX_SLC52A2.vcf.gz.gorz".getBytes(StandardCharsets.UTF_8));
@@ -324,7 +324,7 @@ public void testReadServer() throws IOException {
324324
@Test
325325
public void testProjectWriteRootCLI() throws IOException {
326326
String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET);
327-
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
327+
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
328328
String dataPath = DataUtil.toFile("dummy", DataType.GOR);
329329

330330
runGorPipeCLI("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext);
@@ -341,7 +341,7 @@ public void testProjectWriteRootCLI() throws IOException {
341341
@Test
342342
public void testProjectWriteUserDataCLI() throws IOException {
343343
String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET);
344-
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
344+
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
345345
String dataPath = DataUtil.toFile("user_data/dummy", DataType.GOR);
346346

347347
runGorPipeCLI("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext);
@@ -358,7 +358,7 @@ public void testProjectWriteUserDataCLI() throws IOException {
358358
@Test
359359
public void testProjectWriteRootServer() {
360360
String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET);
361-
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
361+
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
362362
String dataPath = DataUtil.toFile("dummy", DataType.GOR);
363363

364364
var e = Assert.assertThrows("Server context should not allow write to root", GorSecurityException.class, () -> runGorPipeServer("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext));
@@ -368,7 +368,7 @@ public void testProjectWriteRootServer() {
368368
@Test
369369
public void testProjectWriteUserDataServer() throws IOException {
370370
String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET);
371-
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
371+
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
372372
String dataPath = DataUtil.toFile("user_data/dummy", DataType.GOR);
373373

374374
runGorPipeServer("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext);
@@ -385,7 +385,7 @@ public void testProjectWriteUserDataServer() throws IOException {
385385
@Test
386386
public void testProjectWriteUserDataWithIndexServer() throws IOException {
387387
String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET);
388-
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
388+
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
389389
String dataPath = "user_data/dummy.gorz";
390390

391391
runGorPipeServer("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext);
@@ -408,7 +408,7 @@ public void testProjectWriteUserDataWithIndexServer() throws IOException {
408408
@Test
409409
public void testSharedWriteRootServer() {
410410
String securityContext = createSecurityContext("s3region", Credentials.OwnerType.System, "some_env", S3_KEY, S3_SECRET);
411-
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
411+
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
412412
String dataPath = DataUtil.toFile("dummy", DataType.GOR);
413413

414414

@@ -419,7 +419,7 @@ public void testSharedWriteRootServer() {
419419
@Test
420420
public void testSharedWriteUserDataServer() throws IOException {
421421
String securityContext = createSecurityContext("s3region", Credentials.OwnerType.System, "some_env", S3_KEY, S3_SECRET);
422-
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
422+
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
423423
String dataPath = DataUtil.toFile("user_data/dummy", DataType.GOR);
424424

425425
runGorPipeServer("gorrow 1,2,3 | write s3region://shared/" + dataPath, gorRoot, securityContext);
@@ -433,6 +433,51 @@ public void testSharedWriteUserDataServer() throws IOException {
433433
Assert.assertTrue(Files.exists(Path.of(gorRoot, DataUtil.toFile(dataPath, DataType.LINK))));
434434
}
435435

436+
@Test
437+
public void testSharedWriteUserDataServerTeeed() throws IOException {
438+
String securityContext = createSecurityContext("s3region", Credentials.OwnerType.System, "some_env", S3_KEY, S3_SECRET);
439+
String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString();
440+
String dataPath = DataUtil.toFile("user_data/dummy", DataType.GOR);
441+
String dataPath2 = DataUtil.toFile("user_data/dummy2", DataType.GOR);
442+
String dataPath3 = DataUtil.toFile("user_data/dummy3", DataType.GOR);
443+
444+
runGorPipeServer("gorrows -p chr1:10000-10005| tee >(where 1 = 1 | write -c s3region://shared/" + dataPath
445+
+ ") | tee >(where 1 = 1 | write -c s3region://shared/" + dataPath2
446+
+ ") | where 1 = 1 | write -c s3region://shared/" + dataPath3
447+
, gorRoot, securityContext);
448+
449+
runGorPipeServer("gorrows -p chr1:10000-10005| tee >(where 1 = 1 | write -c s3region://shared/" + dataPath
450+
+ ") | tee >(where 1 = 1 | write -c s3region://shared/" + dataPath2
451+
+ ") | where 1 = 1 | write -c s3region://shared/" + dataPath3
452+
, gorRoot, securityContext);
453+
454+
Assert.assertTrue(Files.exists(Path.of(gorRoot, DataUtil.toFile(dataPath, DataType.LINK))));
455+
Assert.assertTrue(Files.exists(Path.of(gorRoot, DataUtil.toFile(dataPath2, DataType.LINK))));
456+
Assert.assertTrue(Files.exists(Path.of(gorRoot, DataUtil.toFile(dataPath3, DataType.LINK))));
457+
458+
// Clean up..
459+
S3SharedSourceProvider provider = new S3RegionSharedSourceProvider();
460+
provider.setConfig(ConfigManager.getPrefixConfig("gor",GorDriverConfig .class));
461+
try(
462+
DataSource source = getDataSourceFromProvider(provider, dataPath, Credentials.OwnerType.System, "some_env"))
463+
464+
{
465+
source.delete();
466+
}
467+
try(
468+
DataSource source = getDataSourceFromProvider(provider, dataPath2, Credentials.OwnerType.System, "some_env"))
469+
470+
{
471+
source.delete();
472+
}
473+
}
474+
475+
476+
477+
/*
478+
| tee >(where depth >= 4 | segspan -maxseg 10000 | write -c ${OUTPUT_WGS_SEG_GOODCOV_4})
479+
| tee >(where depth >= 4 | segspan -maxseg 10000 | hide segCount | join -segseg ${JOIN_EXONS} | write -c ${OUTPUT_WES_SEG_GOODCOV_4})
480+
*/
436481
@Test
437482
public void testProjecSharedProjecttWriteUserDataServer() throws IOException {
438483
String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET);

‎gortools/src/main/java/gorsat/process/GorJavaUtilities.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -498,13 +498,13 @@ public static void writeDictionaryFromMeta(String commandToExecute, FileReader f
498498
} else writeDummyHeader(dictionarypathwriter);
499499
}
500500

501-
localFileReader.writeLinkIfNeeded(dictionarypath);
502-
503501
var linkOptions = LinkFileUtil.extractLinkOptionData(commandToExecute);
504502
if (!Strings.isNullOrEmpty(linkOptions)) {
505503
var linkMetaOption = LinkFileUtil.extractLinkMetaOptionData(commandToExecute);
506504
var linkData = LinkFileUtil.extractLink(fileReader, outfolderpath, linkOptions, linkMetaOption, null);
507505
LinkFileUtil.writeLinkFile(fileReader, linkData);
506+
} else {
507+
localFileReader.writeLinkIfNeeded(dictionarypath);
508508
}
509509
}
510510

‎gortools/src/test/java/gorsat/UTestGorWrite.java‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -178,23 +178,30 @@ public void testWritePathWithExistingBadVersionedLinkFile() throws IOException {
178178

179179
@Test
180180
public void testOverwritePathWithExistingVersionedLinkFile() throws IOException {
181-
Path p = Paths.get("../tests/data/gor/dbsnp_test.gor");
182-
Files.copy(p, workDirPath.resolve("dbsnp.gor"));
183-
184-
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
185-
LinkFile linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gor.link").toString()));
186-
Assert.assertEquals(1, linkFile.getEntriesCount());
181+
var oldAllow = LinkFileV1.allowOverwriteOfTargets;
182+
LinkFileV1.allowOverwriteOfTargets = false;
187183

188-
// Test with same file.
189-
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
190-
linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gor.link").toString()));
191-
Assert.assertEquals(2, linkFile.getEntriesCount());
192-
193-
// Test with different file
194-
Assert.assertThrows( "Overwriting link with same path, throws exception",
195-
GorSystemException.class,
196-
() -> TestUtils.runGorPipe("gor dbsnp.gor | top 1 | write dbsnp2.gor -link dbsnp3.gor",
197-
"-gorroot", workDirPath.toString()));
184+
try {
185+
Path p = Paths.get("../tests/data/gor/dbsnp_test.gor");
186+
Files.copy(p, workDirPath.resolve("dbsnp.gor"));
187+
188+
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
189+
LinkFile linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gor.link").toString()));
190+
Assert.assertEquals(1, linkFile.getEntriesCount());
191+
192+
// Test with same file.
193+
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
194+
linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gor.link").toString()));
195+
Assert.assertEquals(2, linkFile.getEntriesCount());
196+
197+
// Test with different file
198+
Assert.assertThrows("Overwriting link with same path, throws exception",
199+
GorSystemException.class,
200+
() -> TestUtils.runGorPipe("gor dbsnp.gor | top 1 | write dbsnp2.gor -link dbsnp3.gor",
201+
"-gorroot", workDirPath.toString()));
202+
} finally {
203+
LinkFileV1.allowOverwriteOfTargets = oldAllow;
204+
}
198205
}
199206

200207
@Test

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,12 @@ public static LinkFile load(StreamSource source) throws IOException {
6767
return create(source, meta, content);
6868
}
6969

70+
public static LinkFile createOrLoad(StreamSource source, String version) throws IOException {
71+
var content = loadContentFromSource(source);
72+
var meta = LinkFileMeta.createOrLoad(content, version, false);
73+
return create(source, meta, content);
74+
}
75+
7076
public static LinkFile create(StreamSource source, String content) {
7177
var meta = LinkFileMeta.createOrLoad(content, null, true);
7278
return create(source, meta, content);

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public class LinkFileMeta extends BaseMeta {
1717
// Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false).
1818
public static final String HEADER_DATA_LIFECYCLE_MANAGED_KEY = "DATA_LIFECYCLE_MANAGED";
1919

20-
private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", LinkFileV0.VERSION);
20+
static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", LinkFileV0.VERSION);
2121

2222
public static final int DEFAULT_ENTRIES_COUNT_MAX = 100;
2323
public static final long DEFAULT_ENTRIES_AGE_MAX = Long.MAX_VALUE;

0 commit comments

Comments
 (0)