Skip to content

Commit 501e346

Browse files
gmagnugmagnu
authored
fix(ENGKOW-2781): gor support versioning of dictionaries (#93)
* fat(ENGKNOW-2781): Minor tweaks to the MDR error handling. * fat(ENGKNOW-2781): Fix minor issue with Exec. * fat(ENGKNOW-2781): Changes to make -link an pgor/paralle work correctly. * fat(ENGKNOW-2781): Minor refactoring. Cache temp file name for FileTable. * feat(ENGKNOW-2781): Update dict link file support. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Fix invalid sec context. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths. * fix(ENGKNOW-2781): Tweeking link cache paths.
1 parent 9628d16 commit 501e346

8 files changed

Lines changed: 51 additions & 21 deletions

File tree

‎gortools/src/main/java/gorsat/process/GorJavaUtilities.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -432,17 +432,28 @@ public static void createSymbolicLinkSafe(Path resultPath, Path cachePath) throw
432432
*/
433433
public static String verifyLinkFileLastModified(ProjectContext projectContext, String cacheFile) {
434434
if (cacheFile != null && DataUtil.isLink(cacheFile)) {
435+
var invalidCacheFile = false;
435436
try {
436437
var ds = projectContext.getFileReader().resolveUrl(cacheFile);
437438
var linkLastModified = ds.getSourceMetadata().getLinkLastModified();
438439
var lastModified = ds.getSourceMetadata().getLastModified();
439440
if (linkLastModified != null && lastModified > linkLastModified) {
440-
// Delete the link file (from the cache).
441+
// Outdated link file.
442+
invalidCacheFile = true;
443+
}
444+
} catch (Exception e) {
445+
// Can not resolve the file or other errors.
446+
invalidCacheFile = true;
447+
}
448+
449+
if (invalidCacheFile) {
450+
log.debug("Link file {} is out of date and will be re-created.", cacheFile);
451+
try {
441452
Files.delete(Paths.get(cacheFile));
442453
cacheFile = null;
454+
} catch (IOException ioException) {
455+
// Ignore
443456
}
444-
} catch (IOException e) {
445-
// Ignore
446457
}
447458
}
448459
return cacheFile;

‎gortools/src/main/scala/gorsat/Commands/Write.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class Write extends CommandInfo("WRITE",
5151

5252
fileName = if (fileName.isEmpty && linkOpt.nonEmpty) {
5353
val linkMetaInfo = LinkFileUtil.extractLinkMetaInfo(linkMetaOpt)
54-
val linkSourceRef = new SourceReference(linkOpt,
54+
val linkSourceRef = new SourceReference(DataUtil.toLink(linkOpt),
5555
context.getSession.getProjectContext.getFileReader.getSecurityContext,
5656
context.getSession.getProjectContext.getFileReader.getCommonRoot, null, null, true);
5757
// Infer the full file name from the link (and defautl locations)

‎gortools/src/test/java/gorsat/UTestGorWrite.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,7 @@ public void testWriteLinkFileForGordFolderInferFilename() throws IOException {
311311
var linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gord.link").toString()));
312312

313313
Assert.assertEquals(1, linkFile.getEntriesCount());
314-
Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3\\..*?\\.gord/"));
314+
Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3_.*?\\.gord/"));
315315

316316
String linkresult1 = TestUtils.runGorPipe("gor dbsnp.gor| top 1000", "-gorroot", workDirPath.toString());
317317
String linkresult3 = TestUtils.runGorPipe("gor dbsnp3.gord | top 1000", "-gorroot", workDirPath.toString());
@@ -327,7 +327,7 @@ public void testWriteLinkFileForGordFolderInferFilenameParallel() throws IOExcep
327327
var linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gord.link").toString()));
328328

329329
Assert.assertEquals(1, linkFile.getEntriesCount());
330-
Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3\\..*?\\.gord/"));
330+
Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3_.*?\\.gord/"));
331331

332332
String linkresult1 = TestUtils.runGorPipe("gor dbsnp.gor | top 500", "-gorroot", workDirPath.toString());
333333
String linkresult3 = TestUtils.runGorPipe("gor dbsnp3.gord | top 500", "-gorroot", workDirPath.toString());

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,10 @@ public LinkFileMeta getMeta() {
123123
return meta;
124124
}
125125

126+
public int getSerial() {
127+
return meta.getPropertyInt(LinkFileMeta.HEADER_SERIAL_KEY, 0);
128+
}
129+
126130
public String getPath() {
127131
return source.getFullPath();
128132
}

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,23 @@
88
import org.gorpipe.gor.model.FileReader;
99
import org.gorpipe.gor.table.util.PathUtils;
1010
import org.gorpipe.util.Strings;
11+
import org.slf4j.Logger;
1112

1213
import java.io.IOException;
1314
import java.util.regex.Matcher;
1415
import java.util.regex.Pattern;
1516

1617
public class LinkFileUtil {
1718

19+
private static Logger log = org.slf4j.LoggerFactory.getLogger(LinkFileUtil.class);
20+
1821
/**
1922
* Infer the data file name from the link file name.
2023
*
24+
* Notes: The path returned must be idempotent as this is called
25+
* from multiple different places in the code (meaning we
26+
* can not use random or time in the path).
27+
*
2128
* @param linkSource the link file path with the link extension
2229
* @param linkFileMeta additional link file meta data
2330
* @return the data file path
@@ -38,7 +45,7 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
3845
if (!Strings.isNullOrEmpty(linkDataFileParentPath)) {
3946
dataFileParentPath = linkDataFileParentPath;
4047
} else if (link.getLatestEntry() != null) {
41-
dataFileParentPath = PathUtils.getParent(link.getLatestEntryUrl());
48+
dataFileParentPath = PathUtils.getParent(PathUtils.getParent(link.getLatestEntryUrl()));
4249
}
4350

4451
if (!Strings.isNullOrEmpty(linkDataFileParentPath)) {
@@ -59,9 +66,14 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
5966
}
6067
}
6168

62-
var dataFileName = PathUtils.injectRandomStringIntoFileName(PathUtils.getFileName(linkSource.getFullPath()));
69+
var fileName = PathUtils.getFileName(linkSource.getFullPath());
70+
var extraFolder = PathUtils.removeExtensions(fileName);
71+
var uniqueFileName = PathUtils.injectStringIntoFileName(fileName, Integer.toString(link.getSerial() + 1));
72+
73+
log.warn("Inferred file name for link file {} is {}", linkSource.getFullPath(),
74+
PathUtils.resolve(PathUtils.resolve(dataFileParentPath, extraFolder), uniqueFileName));
6375

64-
return PathUtils.resolve(dataFileParentPath, dataFileName);
76+
return PathUtils.resolve(PathUtils.resolve(dataFileParentPath, extraFolder), uniqueFileName);
6577
}
6678

6779
private static Pattern linkPattern = Pattern.compile(".* -link ([^\\s]*) ?.*", Pattern.CASE_INSENSITIVE);

‎model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -348,16 +348,19 @@ public static Path getTempFilePath(Path filePath) {
348348
}
349349

350350
public static String injectRandomStringIntoFileName(String fileName) {
351+
return injectStringIntoFileName(fileName, RandomStringUtils.insecure().next(8, true, true));
352+
}
353+
354+
public static String injectStringIntoFileName(String fileName, String injectString) {
351355
var tempFileName = "";
352-
String uniqId = RandomStringUtils.insecure().next(8, true, true);
353356
var linkPathSplit = fileName.indexOf('.', fileName.indexOf("/"));
354357
if (linkPathSplit > 0) {
355-
tempFileName = "%s.%s.%s".formatted(
358+
tempFileName = "%s_%s.%s".formatted(
356359
fileName.substring(0, linkPathSplit),
357-
uniqId,
360+
injectString,
358361
fileName.substring(linkPathSplit + 1));
359362
} else {
360-
tempFileName = "%s.%s".formatted(fileName, uniqId);
363+
tempFileName = "%s_%s".formatted(fileName, injectString);
361364
}
362365

363366
return tempFileName.replaceAll("\\.link$", "");

‎model/src/main/java/org/gorpipe/gor/util/DataUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ public static String toLinkFile(String name, DataType type) {
126126
}
127127

128128
public static String toLink(String path) {
129-
return PathUtils.stripTrailingSlash(path) + DataType.LINK.suffix;
129+
return DataUtil.isLink(path) ? path : PathUtils.stripTrailingSlash(path) + DataType.LINK.suffix;
130130
}
131131

132132
public static String toVersionedLink(String path) {

‎model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ public void testInferDataFileNameFromLinkFile_FromEnvVariable_WithProject() thro
201201

202202
String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference("x.gor.link", null, "/projects/test", -1, null, null, false, false)), null);
203203
assertNotNull(result);
204-
assertTrue(result.matches((root + "/test/x\\..*\\.gor").replace("/", "\\/")));
204+
assertTrue(result.matches((root + "/test/x/x_.*\\.gor").replace("/", "\\/")));
205205
}
206206

207207
@Test
@@ -210,19 +210,19 @@ public void testInferDataFileNameFromLinkFile_FromEnvVariable_WithOutProject() t
210210
environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_ROOT_URL, root);
211211

212212
String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource("x.gor.link"), null);
213-
assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/")));
213+
assertTrue(result.matches((root + "/x/x_.*\\.gor").replace("/", "\\/")));
214214
}
215215

216216
@Test
217217
public void testInferDataFileNameFromLinkFile_FromExiting_File() throws Exception {
218218
String root = "/managed/fromfile";
219219
String linkFilePath = "x.gor.link";
220220
Files.createDirectory(workPath.resolve("test"));
221-
Files.writeString(workPath.resolve("test").resolve(linkFilePath), root + "/source/y.gorz\n");
221+
Files.writeString(workPath.resolve("test").resolve(linkFilePath), root + "/source/y/y.gorz\n");
222222

223223
String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath, null, workPath.resolve("test").toString(), -1, null, null, false, false)), null);
224224
assertNotNull(result);
225-
assertTrue(result.matches((root + "/source/x\\..*\\.gor").replace("/", "\\/")));
225+
assertTrue(result.matches((root + "/source/x/x_.*\\.gor").replace("/", "\\/")));
226226
}
227227

228228
@Test
@@ -233,7 +233,7 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam() throws Exception {
233233

234234
String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath)), linkFileMeta);
235235
assertNotNull(result);
236-
assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/")));
236+
assertTrue(result.matches((root + "/x/x_..*\\.gor").replace("/", "\\/")));
237237
}
238238

239239
@Test
@@ -247,7 +247,7 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam_ExistingFile() throw
247247

248248
String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath)), linkFileMeta);
249249
assertNotNull(result);
250-
assertTrue(result.matches((paramroot + "/x\\..*\\.gor").replace("/", "\\/")));
250+
assertTrue(result.matches((paramroot + "/x/x_.*\\.gor").replace("/", "\\/")));
251251
}
252252

253253
@Test
@@ -258,6 +258,6 @@ public void testInferDataFileNameFromLinkFile_AbsolutePath() throws Exception {
258258
String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource("/abs/path/x.gor.link"), null);
259259

260260
assertNotNull(result);
261-
assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/")));
261+
assertTrue(result.matches((root + "/x/x_.*\\.gor").replace("/", "\\/")));
262262
}
263263
}

0 commit comments

Comments
 (0)