diff --git a/code/processes/setter.q b/code/processes/setter.q new file mode 100644 index 000000000..6dec4ae12 --- /dev/null +++ b/code/processes/setter.q @@ -0,0 +1,78 @@ +// Setter process to set cache to disk + +\d .anycache.setter + +// Save cache down to disk +savecachedowntodisk:{[data;filepath] (hsym filepath) set data }; + +// Get location of cache config and load it in. +cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; +cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; + +detectandwritecache:{ + cacheinfo:detectcachetobuild[]; + if[not count cacheinfo; + :(::) + ]; + writetoken[cachepath:cacheinfo`cachepath;`start]; + //Included some basic error trapping here + success:@[`generateandwritecache[cachepath];cacheinfo`args;0b]; + if[not success; + cleanupcache cachepath; + :(::) + ]; + writetoken[cachepath;`end]; + //Can eject now if there are still caches to be built in the main cache + if[not count remainingcaches:cacheinfo`maincachepath; + :(::) + ]; + completecache cacheinfo`maincachepath +}; + + writetoken:{[dir;stage] + //accepts start or end and saves the current time as a timestamp to a flat file + //in dir as `:startTime or `:endTime + (` sv (dir;stage)) set .z.P +} + +detectcachetobuild:{ + latestcache:{cachename:"MyFirstCache"; + maincachepath:` sv (hsym `$.setter.cacheconfig.cacheRootDir),`$cachename; + caches:` sv' maincachepath,'(key maincachepath) except `$cachename; + if[0 = count caches; :`cachename`newcache!(cachename,"_",string .z.P;1b)]; + cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches; + if[("N"$.setter.cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)]; + :`cachename`newcache!(cachewithmaxstarttime;0b)}[] + + if[latestcache[`newcache];writetoken[latestcache[`cachename];`start]] + if[latestcache[`newcache]; cachewithmaxstarttime:` sv maincachepath,`$latestcache[`newcache]] + + componentcaches:` sv' cachewithmaxstarttime,/:key .setter.cacheconfig.componentCaches + incompletecomponentcaches:key[d2] where not `end in/: value d2:componentcaches!key each componentcaches + writetoken[;`start] each incompletecomponentcaches; + writetoken[;`setter1] each incompletecomponentcaches; + `maincachepath`cachepath`args!(maincachepath;incompletecomponentcaches;enlist`) +} + +generateandwritecache:{[cachepath; args] + cachename:last ` vs cachepath; + connectiondetails: .anycache.config.componentCaches[cachename].dataSource; + cache: connectiondetails".anycache.sampleanalytic[(::)]"; + .anymap.writetoanymap[cache;cachepath] +} + +cleanupcache:{[cachepath] + //Want to just remove the component cache (cacheName) from the main cache directory in event of a failure + hdel cachepath +} + +completecache:{[maincachepath] + cachename: string last ` vs maincachepath; + latestcache:first system"ls -lt ",(1_string maincachepath), " | grep -vE '(^l|total)' | head -n 1 | awk '{print $NF}'"; + latestcachefilepath: ` sv maincachepath,`$latestcache + writetoken[latestcachefilepath;`end]; + system"ln -sfn ", latestcache, " ", (1_string maincachepath), "/", cachename + hdel each ` sv' maincachepath,'(key maincachepath) except (`$cachename;`$latestcache) +} + +\d . \ No newline at end of file