From eabb34b6010331d093941f9ee47f516a3b6ecb31 Mon Sep 17 00:00:00 2001 From: Benedictus Alvian Sofjan Date: Mon, 20 Jul 2026 09:15:36 +0100 Subject: [PATCH 1/2] Static Cachebuilder --- code/processes/setter.q | 84 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 code/processes/setter.q diff --git a/code/processes/setter.q b/code/processes/setter.q new file mode 100644 index 000000000..d67845fc1 --- /dev/null +++ b/code/processes/setter.q @@ -0,0 +1,84 @@ +// Setter process to set cache to disk + +\d .anycache.setter + +// Save cache down to disk +savecachedowntodisk:{[data;filePath] (hsym filePath) set data }; + +// Get location of cache config and load it in. +cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; +cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; + +\d . + +\d .anycache.setter + +getcachefromdisk:{ .proc.loadf } + +detectAndWriteCache:{ + cacheInfo:detectCacheToBuild[]; + if[not count cacheInfo; + :(::) + ]; + writeToken[cachePath:cacheInfo`cachePath;`start]; + //Included some basic error trapping here + success:@[`generateAndWriteCache[cachePath];cacheInfo`args;0b]; + if[not success; + cleanupCache cachePath; + :(::) + ]; + writeToken[cachePath;`end]; + //Can eject now if there are still caches to be built in the main cache + If[not count remainingCaches:cacheInfo`mainCachePath; + :(::) + ]; + completeCache cacheInfo`mainCachePath +}; + + writeToken:{[dir;stage] + //accepts start or end and saves the current time as a timestamp to a flat file + //in dir as `:startTime or `:endTime + (` sv (dir;stage)) set .z.P +} + +detectCacheToBuild:{ + latestCache:{cacheName:"MyFirstCache"; + mainCachePath:` sv (hsym `$.setter.cacheconfig.cacheRootDir),`$cacheName; + caches:` sv' mainCachePath,'(key mainCachePath) except `$cacheName; + if[0 = count caches; :`cacheName`newCache!(cacheName,"_",string .z.P;1b)]; + cacheWithMaxStartTime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches; + if[("N"$.setter.cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cacheWithMaxStartTime;13 16 19;:;"::."];:`cacheName`newCache!(cacheName,"_",string .z.P;1b)]; + :`cacheName`newCache!(cacheWithMaxStartTime;0b)}[] + + if[latestCache[`newCache];writeToken[latestCache[`cacheName];`start]] + if[latestCache[`newCache]; cacheWithMaxStartTime:` sv mainCachePath,`$latestCache[`newCache]] + + componentCaches:` sv' cacheWithMaxStartTime,/:key .setter.cacheconfig.componentCaches + incompleteComponentCaches:key[d2] where not `end in/: value d2:componentCaches!key each componentCaches + writeToken[;`start] each incompleteComponentCaches; + writeToken[;`setter1] each incompleteComponentCaches; + `mainCachePath`cachePath`args!(mainCachePath;incompleteComponentCaches;enlist`) +} + +generateAndWriteCache:{[cachePath; args] +cachename:last ` vs cachePath; +connectiondetails: .anycache.config.componentCaches[cachename].dataSource; +cache: connectiondetails".anycache.sampleanalytic[(::)]"; +.anymap.writeToAnyMap[cache;cachePath] +} + +cleanupCache:{[cachePath] + //Want to just remove the component cache (cacheName) from the main cache directory in event of a failure + hdel cachePath +} + +completeCache:{[mainCachePath] + cacheName: string last ` vs mainCachePath; + latestCache:first system"ls -lt ",(1_string mainCachePath), " | grep -vE '(^l|total)' | head -n 1 | awk '{print $NF}'"; + latestCacheFilePath: ` sv mainCachePath,`$latestCache + writeToken[latestCacheFilePath;`end]; + system"ln -sfn ", latestCache, " ", (1_string mainCachePath), "/", cacheName + hdel each ` sv' mainCachePath,'(key mainCachePath) except (`$cacheName;`$latestCache) +} + +\d . \ No newline at end of file From 2dc2bd5b37ed0d934244adcb1aac7d5513df26bd Mon Sep 17 00:00:00 2001 From: Benedictus Alvian Sofjan Date: Mon, 27 Jul 2026 09:10:05 +0100 Subject: [PATCH 2/2] Update setter.q --- code/processes/setter.q | 88 +++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 47 deletions(-) diff --git a/code/processes/setter.q b/code/processes/setter.q index d67845fc1..6dec4ae12 100644 --- a/code/processes/setter.q +++ b/code/processes/setter.q @@ -3,82 +3,76 @@ \d .anycache.setter // Save cache down to disk -savecachedowntodisk:{[data;filePath] (hsym filePath) set data }; +savecachedowntodisk:{[data;filepath] (hsym filepath) set data }; // Get location of cache config and load it in. cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; -\d . - -\d .anycache.setter - -getcachefromdisk:{ .proc.loadf } - -detectAndWriteCache:{ - cacheInfo:detectCacheToBuild[]; - if[not count cacheInfo; +detectandwritecache:{ + cacheinfo:detectcachetobuild[]; + if[not count cacheinfo; :(::) ]; - writeToken[cachePath:cacheInfo`cachePath;`start]; + writetoken[cachepath:cacheinfo`cachepath;`start]; //Included some basic error trapping here - success:@[`generateAndWriteCache[cachePath];cacheInfo`args;0b]; + success:@[`generateandwritecache[cachepath];cacheinfo`args;0b]; if[not success; - cleanupCache cachePath; + cleanupcache cachepath; :(::) ]; - writeToken[cachePath;`end]; + writetoken[cachepath;`end]; //Can eject now if there are still caches to be built in the main cache - If[not count remainingCaches:cacheInfo`mainCachePath; + if[not count remainingcaches:cacheinfo`maincachepath; :(::) ]; - completeCache cacheInfo`mainCachePath + completecache cacheinfo`maincachepath }; - writeToken:{[dir;stage] + writetoken:{[dir;stage] //accepts start or end and saves the current time as a timestamp to a flat file //in dir as `:startTime or `:endTime (` sv (dir;stage)) set .z.P } -detectCacheToBuild:{ - latestCache:{cacheName:"MyFirstCache"; - mainCachePath:` sv (hsym `$.setter.cacheconfig.cacheRootDir),`$cacheName; - caches:` sv' mainCachePath,'(key mainCachePath) except `$cacheName; - if[0 = count caches; :`cacheName`newCache!(cacheName,"_",string .z.P;1b)]; - cacheWithMaxStartTime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches; - if[("N"$.setter.cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cacheWithMaxStartTime;13 16 19;:;"::."];:`cacheName`newCache!(cacheName,"_",string .z.P;1b)]; - :`cacheName`newCache!(cacheWithMaxStartTime;0b)}[] +detectcachetobuild:{ + latestcache:{cachename:"MyFirstCache"; + maincachepath:` sv (hsym `$.setter.cacheconfig.cacheRootDir),`$cachename; + caches:` sv' maincachepath,'(key maincachepath) except `$cachename; + if[0 = count caches; :`cachename`newcache!(cachename,"_",string .z.P;1b)]; + cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches; + if[("N"$.setter.cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)]; + :`cachename`newcache!(cachewithmaxstarttime;0b)}[] - if[latestCache[`newCache];writeToken[latestCache[`cacheName];`start]] - if[latestCache[`newCache]; cacheWithMaxStartTime:` sv mainCachePath,`$latestCache[`newCache]] + if[latestcache[`newcache];writetoken[latestcache[`cachename];`start]] + if[latestcache[`newcache]; cachewithmaxstarttime:` sv maincachepath,`$latestcache[`newcache]] - componentCaches:` sv' cacheWithMaxStartTime,/:key .setter.cacheconfig.componentCaches - incompleteComponentCaches:key[d2] where not `end in/: value d2:componentCaches!key each componentCaches - writeToken[;`start] each incompleteComponentCaches; - writeToken[;`setter1] each incompleteComponentCaches; - `mainCachePath`cachePath`args!(mainCachePath;incompleteComponentCaches;enlist`) + componentcaches:` sv' cachewithmaxstarttime,/:key .setter.cacheconfig.componentCaches + incompletecomponentcaches:key[d2] where not `end in/: value d2:componentcaches!key each componentcaches + writetoken[;`start] each incompletecomponentcaches; + writetoken[;`setter1] each incompletecomponentcaches; + `maincachepath`cachepath`args!(maincachepath;incompletecomponentcaches;enlist`) } -generateAndWriteCache:{[cachePath; args] -cachename:last ` vs cachePath; -connectiondetails: .anycache.config.componentCaches[cachename].dataSource; -cache: connectiondetails".anycache.sampleanalytic[(::)]"; -.anymap.writeToAnyMap[cache;cachePath] +generateandwritecache:{[cachepath; args] + cachename:last ` vs cachepath; + connectiondetails: .anycache.config.componentCaches[cachename].dataSource; + cache: connectiondetails".anycache.sampleanalytic[(::)]"; + .anymap.writetoanymap[cache;cachepath] } -cleanupCache:{[cachePath] - //Want to just remove the component cache (cacheName) from the main cache directory in event of a failure - hdel cachePath +cleanupcache:{[cachepath] + //Want to just remove the component cache (cacheName) from the main cache directory in event of a failure + hdel cachepath } -completeCache:{[mainCachePath] - cacheName: string last ` vs mainCachePath; - latestCache:first system"ls -lt ",(1_string mainCachePath), " | grep -vE '(^l|total)' | head -n 1 | awk '{print $NF}'"; - latestCacheFilePath: ` sv mainCachePath,`$latestCache - writeToken[latestCacheFilePath;`end]; - system"ln -sfn ", latestCache, " ", (1_string mainCachePath), "/", cacheName - hdel each ` sv' mainCachePath,'(key mainCachePath) except (`$cacheName;`$latestCache) +completecache:{[maincachepath] + cachename: string last ` vs maincachepath; + latestcache:first system"ls -lt ",(1_string maincachepath), " | grep -vE '(^l|total)' | head -n 1 | awk '{print $NF}'"; + latestcachefilepath: ` sv maincachepath,`$latestcache + writetoken[latestcachefilepath;`end]; + system"ln -sfn ", latestcache, " ", (1_string maincachepath), "/", cachename + hdel each ` sv' maincachepath,'(key maincachepath) except (`$cachename;`$latestcache) } \d . \ No newline at end of file