From c3206c2b40c6551115411ddfec33a8bf1df48c99 Mon Sep 17 00:00:00 2001 From: joaopluigi Date: Thu, 18 Dec 2025 15:32:27 -0300 Subject: [PATCH 1/9] initial draft on profile env execution --- src/nodely/data.clj | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/nodely/data.clj b/src/nodely/data.clj index 0a6bf82..2496b07 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -170,6 +170,38 @@ ([node f] (catch-node node f {}))) +(comment + (require '[criterium.core :as criterium]) + + (defn time-body + [{:keys [k s]} f] + (fn [args] + (let [start# (criterium/timestamp) + ret# (f args) + finish# (criterium/timestamp)] + (swap! s assoc k (- finish# start#)) + ret#))) + + ; (update-node (leaf #{} #(Thread/sleep 1000)) new-value {}) + + (do + (def my-atom (atom {})) + (def updated-node (env-update-helper (leaf #{} (fn [_] (Thread/sleep 1000))) {:k :x :s my-atom} {} time-body)) + + (def update-node-branch (let [condition (leaf #{} (fn [_] (do (Thread/sleep 1000) true))) + truthy (leaf #{} (fn [_] (Thread/sleep 1000))) + falsey (value 20)] + (env-update-helper (branch condition truthy falsey) {:k :y :s my-atom} {} time-body))) + + {:x {:condition 1000 :truthy 1000 :falsey 1000}} + + ((:nodely.data/fn updated-node) {}) + @my-atom + ; + ) +; + ) + ;; ;; Env Utils ;; From 1190c36432a67652d85b9ec6073a1b540a9440b4 Mon Sep 17 00:00:00 2001 From: Sophia Velten Date: Thu, 15 Jan 2026 16:39:21 -0300 Subject: [PATCH 2/9] add claude code solution --- src/nodely/profile.clj | 245 +++++++++++++++++++++++++++++++++++ test/nodely/profile_test.clj | 210 ++++++++++++++++++++++++++++++ 2 files changed, 455 insertions(+) create mode 100644 src/nodely/profile.clj create mode 100644 test/nodely/profile_test.clj diff --git a/src/nodely/profile.clj b/src/nodely/profile.clj new file mode 100644 index 0000000..ccb6bb3 --- /dev/null +++ b/src/nodely/profile.clj @@ -0,0 +1,245 @@ +(ns nodely.profile + "Time profiling utilities for nodely environments. + + Provides non-invasive profiling that wraps existing environments + to record execution timing for each node without modifying the + core execution logic." + (:refer-clojure :exclude [sequence]) + (:require + [nodely.data :as data])) + +;; Forward declaration for conditional core.async support +(declare make-profiled-async-thunk) + +(defn- normalize-path + "Normalizes a path for use as a key. + Single-element paths become simple keywords, multi-element paths stay as vectors." + [path] + (if (= 1 (count path)) + (first path) + path)) + +(defn- record-timing! + "Records elapsed time in nanoseconds to the profile atom at the given path. + Single-element paths are stored as keywords, multi-element as vectors." + [profile-atom path elapsed-ns] + (swap! profile-atom assoc (normalize-path path) {:elapsed-ns elapsed-ns})) + +(defn- wrap-fn-with-timing + "Wraps a function to record its execution time into the profile atom." + [f profile-atom path] + (fn [args] + (let [start (System/nanoTime) + result (f args) + end (System/nanoTime)] + (record-timing! profile-atom path (- end start)) + result))) + +(defn- channel-leaf-fn? + "Checks if a function is an AsyncThunk (channel-leaf)." + [f] + (and (record? f) (contains? f :channel-fn))) + +(defn- profile-leaf + "Wraps a leaf node to record execution time. + Channel-leaf nodes (AsyncThunk) are wrapped specially to handle async timing." + [leaf profile-atom path] + (let [f (::data/fn leaf)] + (if (channel-leaf-fn? f) + (update leaf ::data/fn #(make-profiled-async-thunk % profile-atom path)) + (update leaf ::data/fn #(wrap-fn-with-timing % profile-atom path))))) + +(defn- profile-sequence + "Wraps a sequence node to record execution time of its process-node. + Note: sequence nodes with value process-nodes (simple fns) cannot be profiled + at the individual element level since the fn is applied by the engine." + [sequence-node profile-atom path] + (let [process-node (::data/process-node sequence-node)] + (case (::data/type process-node) + :value sequence-node ; value process-nodes are just functions stored as values + :leaf (update sequence-node ::data/process-node + #(profile-leaf % profile-atom (conj path :process)))))) + +(declare profile-node) + +(defn- profile-branch + "Wraps a branch node to record execution time of condition, truthy, and falsey paths." + [branch profile-atom path] + (-> branch + (update ::data/condition #(profile-node % profile-atom (conj path :condition))) + (update ::data/truthy #(profile-node % profile-atom (conj path :truthy))) + (update ::data/falsey #(profile-node % profile-atom (conj path :falsey))))) + +(defn profile-node + "Recursively wraps a node for profiling. + + For branches, uses path-aware keys to distinguish condition/truthy/falsey. + Values are not wrapped since they're immediate and require no computation." + [node profile-atom path] + (case (::data/type node) + :value node + :leaf (profile-leaf node profile-atom path) + :sequence (profile-sequence node profile-atom path) + :branch (profile-branch node profile-atom path))) + +(defn profile-env + "Transforms an environment for profiling. + + Returns a tuple of [profiled-env profile-atom] where: + - profiled-env: the environment with timing instrumentation + - profile-atom: an atom that will contain timing data after evaluation + + Example usage: + ```clojure + (let [[profiled-env profile-data] (profile-env my-env)] + (nodely/eval-key profiled-env :target {::nodely/engine :sync.lazy}) + @profile-data) + ;; => {:a {:elapsed-ns 1000234} + ;; :b {:elapsed-ns 2003421} + ;; [:c :condition] {:elapsed-ns 500123}} + ``` + + The profile data uses vector paths for nested nodes: + - Top-level leaf: :key -> {:elapsed-ns n} + - Branch condition: [:key :condition] -> {:elapsed-ns n} + - Branch truthy path: [:key :truthy] -> {:elapsed-ns n} + - Nested branches: [:key :truthy :condition] -> {:elapsed-ns n}" + [env] + (let [profile-atom (atom {})] + [(reduce-kv + (fn [acc k node] + (assoc acc k (profile-node node profile-atom [k]))) + {} + env) + profile-atom])) + +(defn profile-env-with-atom + "Like profile-env, but uses a provided atom for collecting profile data. + + Useful when you want to aggregate profiling data across multiple evaluations + or provide your own storage mechanism." + [env profile-atom] + (reduce-kv + (fn [acc k node] + (assoc acc k (profile-node node profile-atom [k]))) + {} + env)) + +(defn total-time + "Calculates the total time from profile data. + + Note: This is the sum of all node times, which may be greater than + wall-clock time if nodes were evaluated in parallel." + [profile-data] + (->> profile-data + vals + (map :elapsed-ns) + (reduce + 0))) + +(defn slowest-nodes + "Returns the n slowest nodes from profile data, sorted by elapsed time descending. + + Each entry is a map with :path and :elapsed-ns keys." + ([profile-data] + (slowest-nodes profile-data 10)) + ([profile-data n] + (->> profile-data + (map (fn [[path timing]] {:path path :elapsed-ns (:elapsed-ns timing)})) + (sort-by :elapsed-ns >) + (take n)))) + +(defn format-timing + "Formats elapsed nanoseconds as a human-readable string." + [elapsed-ns] + (cond + (< elapsed-ns 1000) (str elapsed-ns " ns") + (< elapsed-ns 1000000) (format "%.2f µs" (/ elapsed-ns 1000.0)) + (< elapsed-ns 1000000000) (format "%.2f ms" (/ elapsed-ns 1000000.0)) + :else (format "%.2f s" (/ elapsed-ns 1000000000.0)))) + +(defn summarize + "Returns a human-readable summary of profile data." + [profile-data] + {:total-time (format-timing (total-time profile-data)) + :node-count (count profile-data) + :slowest (->> (slowest-nodes profile-data 5) + (mapv #(update % :elapsed-ns format-timing)))}) + +;; +;; Async profiling support (requires core.async) +;; + +(defn- try-require-async + "Attempts to require core.async namespaces. Returns true if successful." + [] + (try + (require 'clojure.core.async) + (require 'nodely.engine.core-async.core) + true + (catch Exception _ + false))) + +;; Define the ProfiledAsyncThunk record type when core.async is available +(when (try-require-async) + (eval + '(do + (require '[clojure.core.async :as async]) + (require '[nodely.engine.core-async.core :as core-async]) + + (defrecord ProfiledAsyncThunk [channel-fn profile-atom path] + clojure.lang.IFn + (invoke [_ args] + (let [start (System/nanoTime) + result (async/! (:exception-ch opts) val) + (throw (core-async/user-exception val))) + + (nil? val) + (let [ex (ex-info "channel closed unexpectedly" {:channel orig-ch})] + (async/>! (:exception-ch opts) ex) + (throw (core-async/user-exception ex))) + + :else + (do (nodely.profile/record-timing! profile-atom path (- end start)) + (nodely.data/value val)))))) + (catch clojure.lang.ExceptionInfo e + (when-not (core-async/user-exception? e) + (throw e))))))))))) + +(defn make-profiled-async-thunk + "Creates a profiled wrapper for an AsyncThunk that properly handles + both sync (IFn) and async (FnToChannel) execution contexts." + [async-thunk profile-atom path] + (if (and (try-require-async) + (resolve 'nodely.profile/->ProfiledAsyncThunk)) + (let [constructor (resolve 'nodely.profile/->ProfiledAsyncThunk) + channel-fn (:channel-fn async-thunk)] + (constructor channel-fn profile-atom path)) + ;; Fallback - just wrap for sync timing + (let [channel-fn (:channel-fn async-thunk) + value 1)} + [profiled-env profile-atom] (profile/profile-env env)] + (is (map? profiled-env)) + (is (instance? clojure.lang.Atom profile-atom)) + (is (contains? profiled-env :a))))) + +(deftest profile-leaf-records-timing + (testing "profiled leaf nodes record execution time" + (let [env {:a (nodely/>value 1) + :b (nodely/>leaf (inc ?a))} + [profiled-env profile-atom] (profile/profile-env env) + result (nodely/eval-key profiled-env :b {::nodely/engine :sync.lazy})] + (is (= 2 result)) + (is (contains? @profile-atom :b)) + (is (number? (get-in @profile-atom [:b :elapsed-ns]))) + (is (pos? (get-in @profile-atom [:b :elapsed-ns])))))) + +(deftest profile-value-nodes-unchanged + (testing "value nodes are not modified (no timing needed)" + (let [env {:a (nodely/>value 42)} + [profiled-env profile-atom] (profile/profile-env env) + result (nodely/eval-key profiled-env :a {::nodely/engine :sync.lazy})] + (is (= 42 result)) + ;; Value nodes don't record timing since they're immediate + (is (not (contains? @profile-atom :a)))))) + +(deftest profile-branch-records-condition-and-path + (testing "branch nodes record timing for condition and taken path" + (let [env {:x (nodely/>value 4) + :y (nodely/>value 100) + :z (nodely/>if (nodely/>leaf (even? ?x)) + (nodely/>leaf (+ ?x 1)) + (nodely/>leaf ?y))} + [profiled-env profile-atom] (profile/profile-env env) + result (nodely/eval-key profiled-env :z {::nodely/engine :sync.lazy})] + (is (= 5 result)) + ;; Should have timing for condition + (is (contains? @profile-atom [:z :condition])) + (is (pos? (get-in @profile-atom [[:z :condition] :elapsed-ns]))) + ;; Should have timing for truthy path (since x=4 is even) + (is (contains? @profile-atom [:z :truthy])) + (is (pos? (get-in @profile-atom [[:z :truthy] :elapsed-ns]))) + ;; Falsey path should NOT have timing (not evaluated) + (is (not (contains? @profile-atom [:z :falsey])))))) + +(deftest profile-branch-falsey-path + (testing "branch nodes record timing for falsey path when condition is false" + (let [env {:x (nodely/>value 3) + :y (nodely/>value 100) + :z (nodely/>if (nodely/>leaf (even? ?x)) + (nodely/>leaf (+ ?x 1)) + (nodely/>leaf ?y))} + [profiled-env profile-atom] (profile/profile-env env) + result (nodely/eval-key profiled-env :z {::nodely/engine :sync.lazy})] + (is (= 100 result)) + ;; Should have timing for condition + (is (contains? @profile-atom [:z :condition])) + ;; Should have timing for falsey path (since x=3 is odd) + (is (contains? @profile-atom [:z :falsey])) + ;; Truthy path should NOT have timing (not evaluated) + (is (not (contains? @profile-atom [:z :truthy])))))) + +(deftest profile-sequence-with-value-process-node + (testing "sequence nodes with value process-node (simple fn) work but don't profile the fn" + (let [env {:items (nodely/>value [1 2 3]) + :doubled (nodely/>sequence #(* 2 %) ?items)} + [profiled-env profile-atom] (profile/profile-env env) + result (nodely/eval-key profiled-env :doubled {::nodely/engine :sync.lazy})] + ;; Result should still be correct + (is (= [2 4 6] result)) + ;; No timing recorded for value-type process nodes + (is (not (contains? @profile-atom :doubled))) + (is (not (contains? @profile-atom [:doubled :process])))))) + +(deftest profile-nested-branches + (testing "nested branches record timing at each level" + (let [env {:a (nodely/>value true) + :b (nodely/>value false) + :x (nodely/>value 10) + :y (nodely/>value 20) + :z (nodely/>if (nodely/>leaf ?a) + (nodely/>if (nodely/>leaf ?b) + (nodely/>leaf ?x) + (nodely/>leaf ?y)) + (nodely/>value 0))} + [profiled-env profile-atom] (profile/profile-env env) + result (nodely/eval-key profiled-env :z {::nodely/engine :sync.lazy})] + (is (= 20 result)) + ;; Outer condition + (is (contains? @profile-atom [:z :condition])) + ;; Inner branch condition (truthy path of outer) + (is (contains? @profile-atom [:z :truthy :condition])) + ;; Inner falsey result (since b=false) + (is (contains? @profile-atom [:z :truthy :falsey]))))) + +(deftest profile-multiple-leaves + (testing "multiple leaf nodes all record timing" + (let [env {:a (nodely/>value 1) + :b (nodely/>leaf (+ ?a 1)) + :c (nodely/>leaf (+ ?b 1)) + :d (nodely/>leaf (+ ?c 1))} + [profiled-env profile-atom] (profile/profile-env env) + result (nodely/eval-key profiled-env :d {::nodely/engine :sync.lazy})] + (is (= 4 result)) + (is (contains? @profile-atom :b)) + (is (contains? @profile-atom :c)) + (is (contains? @profile-atom :d))))) + +(deftest profile-env-with-atom-uses-provided-atom + (testing "profile-env-with-atom uses the provided atom" + (let [my-atom (atom {:existing :data}) + env {:a (nodely/>value 1) + :b (nodely/>leaf (inc ?a))} + profiled-env (profile/profile-env-with-atom env my-atom) + _ (nodely/eval-key profiled-env :b {::nodely/engine :sync.lazy})] + ;; Should preserve existing data + (is (= :data (:existing @my-atom))) + ;; Should add new timing data + (is (contains? @my-atom :b))))) + +(deftest total-time-sums-all-timings + (testing "total-time returns sum of all elapsed times" + (let [profile-data {[:a] {:elapsed-ns 1000} + [:b] {:elapsed-ns 2000} + [:c :condition] {:elapsed-ns 500}}] + (is (= 3500 (profile/total-time profile-data)))))) + +(deftest slowest-nodes-returns-sorted-results + (testing "slowest-nodes returns nodes sorted by time descending" + (let [profile-data {[:a] {:elapsed-ns 1000} + [:b] {:elapsed-ns 5000} + [:c] {:elapsed-ns 3000} + [:d] {:elapsed-ns 2000}} + slowest (profile/slowest-nodes profile-data 3)] + (is (= 3 (count slowest))) + (is (= [:b] (:path (first slowest)))) + (is (= [:c] (:path (second slowest)))) + (is (= [:d] (:path (nth slowest 2))))))) + +(deftest format-timing-formats-correctly + (testing "format-timing produces human-readable output" + (is (= "500 ns" (profile/format-timing 500))) + (is (= "1.50 µs" (profile/format-timing 1500))) + (is (= "2.50 ms" (profile/format-timing 2500000))) + (is (= "1.50 s" (profile/format-timing 1500000000))))) + +(deftest summarize-returns-summary-map + (testing "summarize returns a useful summary" + (let [profile-data {[:a] {:elapsed-ns 1000000} + [:b] {:elapsed-ns 2000000} + [:c] {:elapsed-ns 3000000}} + summary (profile/summarize profile-data)] + (is (= "6.00 ms" (:total-time summary))) + (is (= 3 (:node-count summary))) + (is (= 3 (count (:slowest summary)))) + (is (= [:c] (:path (first (:slowest summary)))))))) + +(deftest profile-with-slow-operations + (testing "profiling captures meaningful timing for slow operations" + (let [env {:a (nodely/>value 1) + :b (nodely/>leaf (do (Thread/sleep 10) (inc ?a)))} + [profiled-env profile-atom] (profile/profile-env env) + _ (nodely/eval-key profiled-env :b {::nodely/engine :sync.lazy}) + elapsed-ns (get-in @profile-atom [:b :elapsed-ns])] + ;; Should be at least 10ms (10,000,000 ns) + (is (>= elapsed-ns 10000000))))) + +(deftest profile-works-with-core-async-engine + (testing "profiling works with core-async lazy-scheduling engine" + (let [env {:a (nodely/>value 1) + :b (nodely/>leaf (inc ?a)) + :c (nodely/>leaf (+ ?a ?b))} + [profiled-env profile-atom] (profile/profile-env env) + result (nodely/eval-key profiled-env :c {::nodely/engine :core-async.lazy-scheduling})] + (is (= 3 result)) + (is (contains? @profile-atom :b)) + (is (contains? @profile-atom :c))))) + +(def tricky-env + {:a (nodely/>value 1) + :b (nodely/>leaf (inc ?a)) + :d (api/>channel-leaf + (core.async/go + (core.async/leaf (+ ?a ?b))}) + +(deftest profile-tricky-env-with-channel-leaf + (testing "profiling works with tricky-env containing channel-leaf nodes" + (testing "regular leaf nodes in the dependency chain are profiled correctly" + (let [[profiled-env profile-atom] (profile/profile-env tricky-env) + ;; Evaluate :c which doesn't involve the channel-leaf + result (nodely/eval-key profiled-env :d {::nodely/engine :core-async.lazy-scheduling})] + ;; :c depends on :b and :a + ;; Result should be (+ 1 (inc 1)) = 3 + ;; o + ;; + (is (>= (get-in @profile-atom [:d :elapsed-ns]) 2000000000)) + (is (= 4 result)))))) From 484a3bdcee3cac297fd50bf6edc41e9fb5ce8172 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Thu, 18 Jun 2026 14:31:00 -0400 Subject: [PATCH 3/9] add notes from designing profiling --- alex_notes_on_profiling.md | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 alex_notes_on_profiling.md diff --git a/alex_notes_on_profiling.md b/alex_notes_on_profiling.md new file mode 100644 index 0000000..aeafce3 --- /dev/null +++ b/alex_notes_on_profiling.md @@ -0,0 +1,47 @@ +# Nodely Env Profiling Concerns + +## Correctly measuring both sync and async eval + +## Profiling demands a mutable place to put the profiled timings + +## Profiling requires creating a single shot env + +## We didn't scratch at any other interesting opportunities + +# Classes of approach + +## Shove everything in Metadata + +Feels expedient, doesn't feel principled + +## Change the implementation of engines so that they accomodate profiling + +Feels expensive, most principled + +## Sneak in a new implementation of the map data structure that lets us smuggle in timing + +Hybrid feel, we're trying to change timing by changing the data structure we reduce in + +# How to change the engine impls? + +They already embody a bad lower-case p protocol. + +Give them a Protocol, each engine implements the protocol between api.v0 and the engine. + +Add another protocol that affords us the opportunity to capture eval start and end times for profiling (maybe). + +Each engine implements the API protocol and the profiling protocol. + +Can compose profiling choice and engine choice by having a no-op impl of the profiling protocol. + +## key -> value installation per engine + +lazy: assoc the k (node name) onto a derived version of env (derivation from node->value) + +manifold: eager assoc futures to result env, futures are lazily evaled on deref, everything must be dereffed! + +virtual_workers: eager assoc vfutures to result env, futures start running eagerly, everything must be dereffed! + +lazy_scheduling: make a new map, access key lazy initializes k . Get "evaled" map is get all scheduled keys and wait on them all materializing + +applicative: make a new map, access key lazy initialized k . Eval node is extract of the monadic context of that one key, eval full map is poke one key and then extract all monadic contexts lazy initialized From b779e878937ee441dff74d88e51e576ccdcdb9f9 Mon Sep 17 00:00:00 2001 From: joaopluigi Date: Thu, 2 Jul 2026 15:04:46 -0300 Subject: [PATCH 4/9] first protocol for Engine --- src/nodely/engine/lazy.clj | 14 +++++++++++++- src/nodely/engine/protocols.clj | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 src/nodely/engine/protocols.clj diff --git a/src/nodely/engine/lazy.clj b/src/nodely/engine/lazy.clj index 61bb1bd..2e81b36 100644 --- a/src/nodely/engine/lazy.clj +++ b/src/nodely/engine/lazy.clj @@ -3,7 +3,19 @@ (:require [clojure.core.async :as async] [nodely.data :as data] - [nodely.engine.core :as core])) + [nodely.engine.core :as core] + [nodely.engine.protocols :as engine.protocols])) + +(deftype LazyEngine [] + engine.protocols/Engine + (-eval [_engine env k _opts] + (core/resolve k env)) + + (-eval-key [engine env k opts] + (data/get-value (engine.protocols/-eval engine env k opts) k)) + + (-eval-key-channel [engine env k opts] + (async/thread (engine.protocols/-eval-key engine env k opts)))) (defn eval [env k] diff --git a/src/nodely/engine/protocols.clj b/src/nodely/engine/protocols.clj new file mode 100644 index 0000000..a3cba00 --- /dev/null +++ b/src/nodely/engine/protocols.clj @@ -0,0 +1,24 @@ +(ns nodely.engine.protocols) + +(defprotocol Engine + (-eval [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.") + (-eval-key [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.") + (-eval-key-channel [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.")) + +(defn eval + ([engine env k] + (-eval engine env k {})) + ([engine env k opts] + (-eval engine env k opts))) + +(defn eval-key + ([engine env k] + (-eval-key engine env k {})) + ([engine env k opts] + (-eval-key engine env k opts))) + +(defn eval-key-channel + ([engine env k] + (-eval-key-channel engine env k {})) + ([engine env k opts] + (-eval-key-channel engine env k opts))) From 78a225ed72d2bfb86f235bcbbf86eda5abed0d04 Mon Sep 17 00:00:00 2001 From: joaopluigi Date: Thu, 9 Jul 2026 15:31:05 -0300 Subject: [PATCH 5/9] change engine-fn implementation to support protocol engines --- src/nodely/api/v0.clj | 159 ++++++++++++++++++-------------- src/nodely/engine/lazy.clj | 10 +- src/nodely/engine/protocols.clj | 4 +- 3 files changed, 103 insertions(+), 70 deletions(-) diff --git a/src/nodely/api/v0.clj b/src/nodely/api/v0.clj index 58b92dd..a84e259 100644 --- a/src/nodely/api/v0.clj +++ b/src/nodely/api/v0.clj @@ -4,9 +4,10 @@ [nodely.data] [nodely.engine.applicative :as applicative] [nodely.engine.core :as engine-core] - [nodely.engine.lazy] + [nodely.engine.protocols :as engine.protocols] [nodely.syntax :as syntax] - [nodely.vendor.potemkin :refer [import-fn import-vars]])) + [nodely.vendor.potemkin :refer [import-fn import-vars]]) + (:import [nodely.engine.lazy LazyEngine])) (import-vars nodely.syntax/>cond nodely.syntax/>if @@ -29,81 +30,83 @@ (def virtual-future-failure (delay - (try (import java.util.concurrent.ThreadPerTaskExecutor) - (require 'nodely.engine.virtual-workers - 'nodely.engine.applicative.virtual-future) - (catch Exception e - {:msg "Classloader could not locate `java.util.concurrent.ThreadPerTaskExecutor`, virtual futures require JDK 21 or higher." - ::error :missing-class - ::requested-class "java.util.concurrent.ThreadPerTaskExecutor" - :cause e})))) + (try (import java.util.concurrent.ThreadPerTaskExecutor) + (require 'nodely.engine.virtual-workers + 'nodely.engine.applicative.virtual-future) + (catch Exception e + {:msg "Classloader could not locate `java.util.concurrent.ThreadPerTaskExecutor`, virtual futures require JDK 21 or higher." + ::error :missing-class + ::requested-class "java.util.concurrent.ThreadPerTaskExecutor" + :cause e})))) (def core-async-failure (delay - (try (require 'nodely.engine.applicative.core-async - 'nodely.engine.core-async.core - 'nodely.engine.core-async.iterative-scheduling - 'nodely.engine.core-async.lazy-scheduling) - (catch Exception e - {:msg "Could not locate core-async on classpath." - ::error :missing-ns - ::requested-namespaces '[nodely.engine.applicative.core-async - nodely.engine.core-async.core - nodely.engine.core-async.iterative-scheduling - nodely.engine.core-async.lazy-scheduling] - :cause e})))) + (try (require 'nodely.engine.applicative.core-async + 'nodely.engine.core-async.core + 'nodely.engine.core-async.iterative-scheduling + 'nodely.engine.core-async.lazy-scheduling) + (catch Exception e + {:msg "Could not locate core-async on classpath." + ::error :missing-ns + ::requested-namespaces '[nodely.engine.applicative.core-async + nodely.engine.core-async.core + nodely.engine.core-async.iterative-scheduling + nodely.engine.core-async.lazy-scheduling] + :cause e})))) (def manifold-failure (delay - (try (require 'nodely.engine.manifold) - (catch Exception e - {:msg "Could not locate manifold on classpath." - ::error :missing-ns - ::requested-namespaces '[nodely.engine.manifold] - :cause e})))) + (try (require 'nodely.engine.manifold) + (catch Exception e + {:msg "Could not locate manifold on classpath." + ::error :missing-ns + ::requested-namespaces '[nodely.engine.manifold] + :cause e})))) (def promesa-failure (delay - (try (require 'nodely.engine.applicative.promesa) - (catch Exception e - {:msg "Could not locate promesa on classpath." - ::error :missing-ns - ::requested-namespaces '[nodely.engine.applicative.promesa] - :cause e})))) + (try (require 'nodely.engine.applicative.promesa) + (catch Exception e + {:msg "Could not locate promesa on classpath." + ::error :missing-ns + ::requested-namespaces '[nodely.engine.applicative.promesa] + :cause e})))) (def engine-data - {:core-async.lazy-scheduling {::ns-name 'nodely.engine.core-async.lazy-scheduling - ::opts-fn identity - ::enable-deref core-async-failure - ::eval-key-channel true} - :core-async.iterative-scheduling {::ns-name 'nodely.engine.core-async.iterative-scheduling - ::opts-fn identity - ::enable-deref core-async-failure} - :async.manifold {::ns-name 'nodely.engine.manifold - ::opts-fn (constantly nil) - ::enable-deref manifold-failure} - :applicative.promesa {::ns-name 'nodely.engine.applicative - ::opts-fn #(assoc % ::applicative/context - (var-get (resolve 'nodely.engine.applicative.promesa/context))) - ::enable-deref promesa-failure} - :applicative.core-async {::ns-name 'nodely.engine.applicative - ::opts-fn #(assoc % ::applicative/context - (var-get (resolve 'nodely.engine.applicative.core-async/context))) - ::eval-key-channel true - ::enable-deref core-async-failure} - :sync.lazy {::ns-name 'nodely.engine.lazy - ::opts-fn (constantly nil) - ::eval-key-channel true - ::enable-deref (delay nil)} - :async.virtual-futures {::ns-name 'nodely.engine.virtual-workers - ::opts-fn (constantly nil) - ::eval-key-channel true - ::enable-deref virtual-future-failure} - :applicative.virtual-future {::ns-name 'nodely.engine.applicative - ::opts-fn #(assoc % ::applicative/context - (var-get (resolve 'nodely.engine.applicative.virtual-future/context))) - ::eval-key-channel true - ::enable-deref virtual-future-failure}}) + {:core-async.lazy-scheduling {::ns-name 'nodely.engine.core-async.lazy-scheduling + ::opts-fn identity + ::enable-deref core-async-failure + ::eval-key-channel true} + :core-async.iterative-scheduling {::ns-name 'nodely.engine.core-async.iterative-scheduling + ::opts-fn identity + ::enable-deref core-async-failure} + :async.manifold {::ns-name 'nodely.engine.manifold + ::opts-fn (constantly nil) + ::enable-deref manifold-failure} + :applicative.promesa {::ns-name 'nodely.engine.applicative + ::opts-fn #(assoc % ::applicative/context + (var-get (resolve 'nodely.engine.applicative.promesa/context))) + ::enable-deref promesa-failure} + :applicative.core-async {::ns-name 'nodely.engine.applicative + ::opts-fn #(assoc % ::applicative/context + (var-get (resolve 'nodely.engine.applicative.core-async/context))) + ::eval-key-channel true + ::enable-deref core-async-failure} + :sync.lazy {::protocol-engine? true + ::instance-constructor #(LazyEngine.) + ; ::ns-name 'nodely.engine.lazy + ::opts-fn (constantly nil) + ; ::eval-key-channel true + } + :async.virtual-futures {::ns-name 'nodely.engine.virtual-workers + ::opts-fn (constantly nil) + ::eval-key-channel true + ::enable-deref virtual-future-failure} + :applicative.virtual-future {::ns-name 'nodely.engine.applicative + ::opts-fn #(assoc % ::applicative/context + (var-get (resolve 'nodely.engine.applicative.virtual-future/context))) + ::eval-key-channel true + ::enable-deref virtual-future-failure}}) (defmacro >channel-leaf [expr] @@ -115,8 +118,7 @@ (mapv #'syntax/question-mark->keyword symbols-to-be-replaced) fn-expr)))) -(defn- engine-fn - [engine-name use] +(defn- data-engine-function [engine-name use] (if-let [engine-data (engine-data engine-name)] (if-let [{:keys [msg cause] :as enable-failure} @(::enable-deref engine-data)] (throw (ex-info msg @@ -129,6 +131,27 @@ {:specified-engine-name engine-name :supported-engine-names (set (keys engine-data))})))) +(defn- protocol-engine-function [engine-name use] + (let [engine-data (engine-data engine-name) + engine-instance ((::instance-constructor engine-data)) + use-to-protocol-fn {'eval engine.protocols/eval + 'eval-key engine.protocols/eval-key + 'eval-key-channel engine.protocols/eval-key-channel}] + (if-let [{:keys [msg cause] :as enable-failure} @(engine.protocols/-enable-deref engine-instance)] + (throw (ex-info msg + (-> enable-failure + (dissoc :msg :cause) + (assoc ::specified-engine-name engine-name)) + cause)) + (partial (use-to-protocol-fn use) engine-instance)))) + +(defn- engine-fn + [engine-name use] + (let [engine-data (engine-data engine-name)] + (if (::protocol-engine? engine-data) + (protocol-engine-function engine-name use) + (data-engine-function engine-name use)))) + (def engine-fn (memoize engine-fn)) (defn eval diff --git a/src/nodely/engine/lazy.clj b/src/nodely/engine/lazy.clj index 2e81b36..20efb61 100644 --- a/src/nodely/engine/lazy.clj +++ b/src/nodely/engine/lazy.clj @@ -6,6 +6,8 @@ [nodely.engine.core :as core] [nodely.engine.protocols :as engine.protocols])) +(defonce enable-deref (delay nil)) + (deftype LazyEngine [] engine.protocols/Engine (-eval [_engine env k _opts] @@ -15,7 +17,13 @@ (data/get-value (engine.protocols/-eval engine env k opts) k)) (-eval-key-channel [engine env k opts] - (async/thread (engine.protocols/-eval-key engine env k opts)))) + (async/thread (engine.protocols/-eval-key engine env k opts))) + + (-eval-key-channel-supported? [_engine] + true) + + (-enable-deref [_engine] + enable-deref)) (defn eval [env k] diff --git a/src/nodely/engine/protocols.clj b/src/nodely/engine/protocols.clj index a3cba00..48a8751 100644 --- a/src/nodely/engine/protocols.clj +++ b/src/nodely/engine/protocols.clj @@ -3,7 +3,9 @@ (defprotocol Engine (-eval [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.") (-eval-key [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.") - (-eval-key-channel [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.")) + (-eval-key-channel [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.") + (-eval-key-channel-supported? [engine] "Applies function f to the value(s) inside the context of the functor fv.") + (-enable-deref [engine] "Please claude update the docstring")) (defn eval ([engine env k] From f9cfecd66a2824ee3eeb90b2c7cf4d542c20f361 Mon Sep 17 00:00:00 2001 From: joaopluigi Date: Thu, 16 Jul 2026 14:39:42 -0300 Subject: [PATCH 6/9] fix LazyEngine require --- src/nodely/api/v0.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nodely/api/v0.clj b/src/nodely/api/v0.clj index a84e259..f870574 100644 --- a/src/nodely/api/v0.clj +++ b/src/nodely/api/v0.clj @@ -2,12 +2,12 @@ (:refer-clojure :exclude [cond eval sequence]) (:require [nodely.data] + [nodely.engine.lazy :as engine.lazy] [nodely.engine.applicative :as applicative] [nodely.engine.core :as engine-core] [nodely.engine.protocols :as engine.protocols] [nodely.syntax :as syntax] - [nodely.vendor.potemkin :refer [import-fn import-vars]]) - (:import [nodely.engine.lazy LazyEngine])) + [nodely.vendor.potemkin :refer [import-fn import-vars]])) (import-vars nodely.syntax/>cond nodely.syntax/>if @@ -93,7 +93,7 @@ ::eval-key-channel true ::enable-deref core-async-failure} :sync.lazy {::protocol-engine? true - ::instance-constructor #(LazyEngine.) + ::instance-constructor #(engine.lazy/->LazyEngine) ; ::ns-name 'nodely.engine.lazy ::opts-fn (constantly nil) ; ::eval-key-channel true From 749f584e9a70495518d7e5b5eb9f87c5b58f957f Mon Sep 17 00:00:00 2001 From: joaopluigi Date: Thu, 16 Jul 2026 14:43:13 -0300 Subject: [PATCH 7/9] fix lint --- src/nodely/api/v0.clj | 66 ++++++++++++++++++------------------ src/nodely/data.clj | 2 +- test/nodely/profile_test.clj | 19 +++++------ 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/src/nodely/api/v0.clj b/src/nodely/api/v0.clj index f870574..3614f00 100644 --- a/src/nodely/api/v0.clj +++ b/src/nodely/api/v0.clj @@ -2,9 +2,9 @@ (:refer-clojure :exclude [cond eval sequence]) (:require [nodely.data] - [nodely.engine.lazy :as engine.lazy] [nodely.engine.applicative :as applicative] [nodely.engine.core :as engine-core] + [nodely.engine.lazy :as engine.lazy] [nodely.engine.protocols :as engine.protocols] [nodely.syntax :as syntax] [nodely.vendor.potemkin :refer [import-fn import-vars]])) @@ -30,47 +30,47 @@ (def virtual-future-failure (delay - (try (import java.util.concurrent.ThreadPerTaskExecutor) - (require 'nodely.engine.virtual-workers - 'nodely.engine.applicative.virtual-future) - (catch Exception e - {:msg "Classloader could not locate `java.util.concurrent.ThreadPerTaskExecutor`, virtual futures require JDK 21 or higher." - ::error :missing-class - ::requested-class "java.util.concurrent.ThreadPerTaskExecutor" - :cause e})))) + (try (import java.util.concurrent.ThreadPerTaskExecutor) + (require 'nodely.engine.virtual-workers + 'nodely.engine.applicative.virtual-future) + (catch Exception e + {:msg "Classloader could not locate `java.util.concurrent.ThreadPerTaskExecutor`, virtual futures require JDK 21 or higher." + ::error :missing-class + ::requested-class "java.util.concurrent.ThreadPerTaskExecutor" + :cause e})))) (def core-async-failure (delay - (try (require 'nodely.engine.applicative.core-async - 'nodely.engine.core-async.core - 'nodely.engine.core-async.iterative-scheduling - 'nodely.engine.core-async.lazy-scheduling) - (catch Exception e - {:msg "Could not locate core-async on classpath." - ::error :missing-ns - ::requested-namespaces '[nodely.engine.applicative.core-async - nodely.engine.core-async.core - nodely.engine.core-async.iterative-scheduling - nodely.engine.core-async.lazy-scheduling] - :cause e})))) + (try (require 'nodely.engine.applicative.core-async + 'nodely.engine.core-async.core + 'nodely.engine.core-async.iterative-scheduling + 'nodely.engine.core-async.lazy-scheduling) + (catch Exception e + {:msg "Could not locate core-async on classpath." + ::error :missing-ns + ::requested-namespaces '[nodely.engine.applicative.core-async + nodely.engine.core-async.core + nodely.engine.core-async.iterative-scheduling + nodely.engine.core-async.lazy-scheduling] + :cause e})))) (def manifold-failure (delay - (try (require 'nodely.engine.manifold) - (catch Exception e - {:msg "Could not locate manifold on classpath." - ::error :missing-ns - ::requested-namespaces '[nodely.engine.manifold] - :cause e})))) + (try (require 'nodely.engine.manifold) + (catch Exception e + {:msg "Could not locate manifold on classpath." + ::error :missing-ns + ::requested-namespaces '[nodely.engine.manifold] + :cause e})))) (def promesa-failure (delay - (try (require 'nodely.engine.applicative.promesa) - (catch Exception e - {:msg "Could not locate promesa on classpath." - ::error :missing-ns - ::requested-namespaces '[nodely.engine.applicative.promesa] - :cause e})))) + (try (require 'nodely.engine.applicative.promesa) + (catch Exception e + {:msg "Could not locate promesa on classpath." + ::error :missing-ns + ::requested-namespaces '[nodely.engine.applicative.promesa] + :cause e})))) (def engine-data {:core-async.lazy-scheduling {::ns-name 'nodely.engine.core-async.lazy-scheduling diff --git a/src/nodely/data.clj b/src/nodely/data.clj index 2496b07..d647188 100644 --- a/src/nodely/data.clj +++ b/src/nodely/data.clj @@ -191,7 +191,7 @@ (def update-node-branch (let [condition (leaf #{} (fn [_] (do (Thread/sleep 1000) true))) truthy (leaf #{} (fn [_] (Thread/sleep 1000))) falsey (value 20)] - (env-update-helper (branch condition truthy falsey) {:k :y :s my-atom} {} time-body))) + (env-update-helper (branch condition truthy falsey) {:k :y :s my-atom} {} time-body))) {:x {:condition 1000 :truthy 1000 :falsey 1000}} diff --git a/test/nodely/profile_test.clj b/test/nodely/profile_test.clj index 246cc2a..35f40e8 100644 --- a/test/nodely/profile_test.clj +++ b/test/nodely/profile_test.clj @@ -1,9 +1,8 @@ (ns nodely.profile-test (:require + [clojure.core.async :as core.async] [clojure.test :refer [deftest is testing]] [nodely.api.v0 :as nodely] - [nodely.data :as data] - [clojure.core.async :as core.async] [nodely.api.v0 :as api] [nodely.profile :as profile])) @@ -40,8 +39,8 @@ (let [env {:x (nodely/>value 4) :y (nodely/>value 100) :z (nodely/>if (nodely/>leaf (even? ?x)) - (nodely/>leaf (+ ?x 1)) - (nodely/>leaf ?y))} + (nodely/>leaf (+ ?x 1)) + (nodely/>leaf ?y))} [profiled-env profile-atom] (profile/profile-env env) result (nodely/eval-key profiled-env :z {::nodely/engine :sync.lazy})] (is (= 5 result)) @@ -59,8 +58,8 @@ (let [env {:x (nodely/>value 3) :y (nodely/>value 100) :z (nodely/>if (nodely/>leaf (even? ?x)) - (nodely/>leaf (+ ?x 1)) - (nodely/>leaf ?y))} + (nodely/>leaf (+ ?x 1)) + (nodely/>leaf ?y))} [profiled-env profile-atom] (profile/profile-env env) result (nodely/eval-key profiled-env :z {::nodely/engine :sync.lazy})] (is (= 100 result)) @@ -90,10 +89,10 @@ :x (nodely/>value 10) :y (nodely/>value 20) :z (nodely/>if (nodely/>leaf ?a) - (nodely/>if (nodely/>leaf ?b) - (nodely/>leaf ?x) - (nodely/>leaf ?y)) - (nodely/>value 0))} + (nodely/>if (nodely/>leaf ?b) + (nodely/>leaf ?x) + (nodely/>leaf ?y)) + (nodely/>value 0))} [profiled-env profile-atom] (profile/profile-env env) result (nodely/eval-key profiled-env :z {::nodely/engine :sync.lazy})] (is (= 20 result)) From cb08ba4b6ed33517112cb17b5aa43b8c483ae0fb Mon Sep 17 00:00:00 2001 From: joaopluigi Date: Thu, 16 Jul 2026 15:20:22 -0300 Subject: [PATCH 8/9] added protocol -prepare-opts to be the ::opts-fn, added docstrings to protocol functions and change eval-* implementations to handle protocol engines --- src/nodely/api/v0.clj | 86 +++++++++++++++------------------ src/nodely/engine/lazy.clj | 5 +- src/nodely/engine/protocols.clj | 11 +++-- 3 files changed, 48 insertions(+), 54 deletions(-) diff --git a/src/nodely/api/v0.clj b/src/nodely/api/v0.clj index 3614f00..2f9aeeb 100644 --- a/src/nodely/api/v0.clj +++ b/src/nodely/api/v0.clj @@ -93,11 +93,8 @@ ::eval-key-channel true ::enable-deref core-async-failure} :sync.lazy {::protocol-engine? true - ::instance-constructor #(engine.lazy/->LazyEngine) - ; ::ns-name 'nodely.engine.lazy - ::opts-fn (constantly nil) - ; ::eval-key-channel true - } + ::instance-constructor engine.lazy/->LazyEngine + ::eval-key-channel true} :async.virtual-futures {::ns-name 'nodely.engine.virtual-workers ::opts-fn (constantly nil) ::eval-key-channel true @@ -131,62 +128,55 @@ {:specified-engine-name engine-name :supported-engine-names (set (keys engine-data))})))) -(defn- protocol-engine-function [engine-name use] - (let [engine-data (engine-data engine-name) - engine-instance ((::instance-constructor engine-data)) - use-to-protocol-fn {'eval engine.protocols/eval - 'eval-key engine.protocols/eval-key - 'eval-key-channel engine.protocols/eval-key-channel}] - (if-let [{:keys [msg cause] :as enable-failure} @(engine.protocols/-enable-deref engine-instance)] - (throw (ex-info msg - (-> enable-failure - (dissoc :msg :cause) - (assoc ::specified-engine-name engine-name)) - cause)) - (partial (use-to-protocol-fn use) engine-instance)))) - -(defn- engine-fn - [engine-name use] - (let [engine-data (engine-data engine-name)] - (if (::protocol-engine? engine-data) - (protocol-engine-function engine-name use) - (data-engine-function engine-name use)))) - -(def engine-fn (memoize engine-fn)) +(def engine-fn (memoize data-engine-function)) (defn eval ([env k] (eval env k {})) - ([env k {engine ::engine - :or {engine :core-async.lazy-scheduling} - :as opts}] - - (let [efn (engine-fn engine 'eval)] - (if-let [opts ((::opts-fn (engine-data engine)) opts)] - (efn env k opts) - (efn env k))))) + ([env k {engine-name ::engine + :or {engine-name :core-async.lazy-scheduling} + :as opts}] + (let [engine-data (engine-data engine-name) + protocol-engine? (::protocol-engine? engine-data) + engine (when protocol-engine? ((::instance-constructor engine-data)))] + (if protocol-engine? + (engine.protocols/eval engine env k (engine.protocols/-prepare-opts engine opts)) + (let [efn (engine-fn engine-name 'eval)] + (if-let [opts ((::opts-fn engine-data) opts)] + (efn env k opts) + (efn env k))))))) (defn eval-key ([env k] (eval-key env k {})) - ([env k {engine ::engine - :or {engine :core-async.lazy-scheduling} - :as opts}] - (let [efn (engine-fn engine 'eval-key)] - (if-let [opts ((::opts-fn (engine-data engine)) opts)] - (efn env k opts) - (efn env k))))) + ([env k {engine-name ::engine + :or {engine-name :core-async.lazy-scheduling} + :as opts}] + (let [engine-data (engine-data engine-name) + protocol-engine? (::protocol-engine? engine-data) + engine (when protocol-engine? ((::instance-constructor engine-data)))] + (if protocol-engine? + (engine.protocols/eval-key engine env k (engine.protocols/-prepare-opts engine opts)) + (let [efn (engine-fn engine-name 'eval-key)] + (if-let [opts ((::opts-fn engine-data) opts)] + (efn env k opts) + (efn env k))))))) (defn eval-key-channel ([env k] (eval-key-channel env k {})) - ([env k {engine ::engine - :or {engine :core-async.lazy-scheduling} + ([env k {engine-name ::engine + :or {engine-name :core-async.lazy-scheduling} :as opts}] - (let [efn (engine-fn engine 'eval-key-channel)] - (if-let [opts ((::opts-fn (engine-data engine)) opts)] - (efn env k opts) - (efn env k))))) + (let [engine-data (engine-data engine-name) + protocol-engine? (::protocol-engine? engine-data) + engine (when protocol-engine? ((::instance-constructor engine-data)))] + (if protocol-engine? + (engine.protocols/eval-key-channel engine env k (engine.protocols/-prepare-opts engine opts)) + (let [efn (engine-fn engine-name 'eval-key-channel)] + (if-let [opts ((::opts-fn engine-data) opts)] + (efn env k opts) + (efn env k))))))) (defn eval-node ([env node] diff --git a/src/nodely/engine/lazy.clj b/src/nodely/engine/lazy.clj index 20efb61..53dc24a 100644 --- a/src/nodely/engine/lazy.clj +++ b/src/nodely/engine/lazy.clj @@ -23,7 +23,10 @@ true) (-enable-deref [_engine] - enable-deref)) + enable-deref) + + (-prepare-opts [_engine _opts] + nil)) (defn eval [env k] diff --git a/src/nodely/engine/protocols.clj b/src/nodely/engine/protocols.clj index 48a8751..0c21298 100644 --- a/src/nodely/engine/protocols.clj +++ b/src/nodely/engine/protocols.clj @@ -1,11 +1,12 @@ (ns nodely.engine.protocols) (defprotocol Engine - (-eval [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.") - (-eval-key [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.") - (-eval-key-channel [engine env k opts] "Applies function f to the value(s) inside the context of the functor fv.") - (-eval-key-channel-supported? [engine] "Applies function f to the value(s) inside the context of the functor fv.") - (-enable-deref [engine] "Please claude update the docstring")) + (-eval [engine env k opts] "Resolves node `k` in `env` using `engine`, returning the resulting env with `k` and its dependencies filled in with their evaluated values.") + (-eval-key [engine env k opts] "Resolves node `k` in `env` using `engine`, returning just the evaluated value of `k`.") + (-eval-key-channel [engine env k opts] "Resolves node `k` in `env` using `engine`, returning a core.async channel that yields the evaluated value of `k`.") + (-eval-key-channel-supported? [engine] "Returns true if `engine` supports `-eval-key-channel`, false otherwise.") + (-enable-deref [engine] "Returns a delay that yields nil if `engine` is available for use, or a map describing why it could not be enabled (e.g. a missing optional dependency) otherwise.") + (-prepare-opts [engine opts] "Transforms the caller-supplied `opts` map into the opts map expected by `engine`'s")) (defn eval ([engine env k] From de8bbdfde2a53aec704beaca115208a3ac3a8f25 Mon Sep 17 00:00:00 2001 From: Alex Redington Date: Thu, 30 Jul 2026 14:48:46 -0400 Subject: [PATCH 9/9] Migrate lazy core async to be a protocol engine. Coded by Claude, planned and reviewed by Alex & Joao --- CLAUDE.md | 220 ++++++++++++++++++ src/nodely/api/v0.clj | 30 ++- .../core_async/lazy_scheduling_engine.clj | 52 +++++ test/nodely/api_test.clj | 6 +- 4 files changed, 297 insertions(+), 11 deletions(-) create mode 100644 CLAUDE.md create mode 100644 src/nodely/engine/core_async/lazy_scheduling_engine.clj diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..10e99e4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,220 @@ +# Migrating an engine from a data-map to the `Engine` protocol + +This guide is for continuing an in-progress refactor: converting each entry in +`nodely.api.v0/engine-data` from a plain data map into a type that implements +`nodely.engine.protocols/Engine`. + +Two engines are already migrated — read them as worked examples before you +start: + +- `:sync.lazy` → `nodely.engine.lazy/LazyEngine` — the **simple** case + (no optional dependency). +- `:core-async.lazy-scheduling` → + `nodely.engine.core-async.lazy-scheduling-engine/CoreAsyncLazySchedulingEngine` + — the **optional-dependency** case (the full pattern). + +Do **one** engine at a time. Run the tests after each. Do not try to do several +at once. + +--- + +## The protocol + +`nodely.engine.protocols/Engine` has six methods. Every migrated engine must +implement all six: + +```clojure +(-eval [engine env k opts]) ; -> resolved env +(-eval-key [engine env k opts]) ; -> value of k +(-eval-key-channel [engine env k opts]) ; -> channel yielding value of k +(-eval-key-channel-supported? [engine]) ; -> true/false +(-enable-deref [engine]) ; -> a delay (see below) +(-prepare-opts [engine opts]) ; -> opts to pass to -eval* +``` + +The dispatch in `v0.clj` already routes any entry that has +`::protocol-engine? true`. You do not need to change `protocols.clj`. + +--- + +## Before you write code: classify the engine + +Look at the engine's existing entry in `engine-data`. **Does it have an +`::enable-deref` key?** + +- **NO `::enable-deref`** (only `:sync.lazy` today) → the engine has no optional + dependency. Put the `deftype` directly in the engine's implementation + namespace, and make `-enable-deref` return `(delay nil)`. Copy `LazyEngine`. + +- **HAS `::enable-deref`** (every other engine) → the engine depends on an + **optional (`:scope "provided"`) library** (core.async, manifold, promesa, + virtual-futures). This is the case with the pitfalls. Use the facade-namespace + pattern below. Copy `CoreAsyncLazySchedulingEngine`. + +--- + +## The optional-dependency pattern (facade namespace) + +### PITFALL 1 — the deftype must load WITHOUT the optional dependency + +`api/v0.clj` is always loaded, even when the optional library is absent. If the +deftype lived in a namespace that `(:require [clojure.core.async ...])` (or +manifold/promesa), then `v0.clj` referencing it would fail to load the moment +the library is missing. That defeats nodely's "works without the optional deps" +contract. + +**DO:** create a new, small **facade namespace** whose only `:require` is +`nodely.engine.protocols`. It must NOT require the optional library, and must +NOT require any namespace that transitively requires it. Load the real +implementation **lazily**, inside the method bodies. See +`lazy_scheduling_engine.clj` for the exact shape: + +```clojure +(ns nodely.engine..-engine + (:require [nodely.engine.protocols :as engine.protocols])) + +(def ^:private impl-ns 'nodely.engine..) ; the real impl + +(def enable-deref + (delay + (try (require impl-ns) nil + (catch Exception e + {:msg "Could not locate on classpath." :cause e})))) + +(defn- impl [fn-name] + (requiring-resolve (symbol (name impl-ns) (name fn-name)))) + +(deftype [] + engine.protocols/Engine + (-eval [_ env k opts] ((impl 'eval) env k opts)) + (-eval-key [_ env k opts] ((impl 'eval-key) env k opts)) + (-eval-key-channel [_ env k opts] ((impl 'eval-key-channel) env k opts)) + (-eval-key-channel-supported? [_] ) + (-enable-deref [_] enable-deref) + (-prepare-opts [_ opts] )) +``` + +### PITFALL 2 — `-enable-deref` is the real gate; the order is construct → check → eval + +`-enable-deref` answers "can this engine run on this classpath?". Because you +must construct the instance before you can call a method on it, and because the +facade constructs without the optional library, the correct order is: + +1. **construct** the engine instance, +2. **deref `-enable-deref`**; if it returns a failure map, throw, +3. only then call `-eval` / `-eval-key` / `-eval-key-channel`. + +The dispatch already does this through the `protocol-engine` helper in `v0.clj`. +You do not need to re-derive it — just make sure your `-enable-deref` returns a +`delay` that attempts `(require impl-ns)` and reports the failure. + +Do NOT try to check availability *before* constructing, and do NOT put the +availability check anywhere that requires loading the optional library first. + +### PITFALL 3 — `-prepare-opts` must reproduce the old `::opts-fn` + +Look at the engine's current `::opts-fn` and make `-prepare-opts` do the same +thing. Do **not** blindly copy `LazyEngine`'s `nil`. + +- `::opts-fn identity` → `(-prepare-opts [_ opts] opts)` +- `::opts-fn (constantly nil)` → `(-prepare-opts [_ _opts] nil)` +- `::opts-fn #(assoc % ::applicative/context ...)` → reproduce that `assoc` + inside `-prepare-opts` (and note the `(resolve '...context)` runs lazily, + which is fine because `-enable-deref` has already confirmed the library is + present by the time `-prepare-opts` is called). + +--- + +## Editing `api/v0.clj` + +1. `:require` the new facade namespace (keep the require list alphabetical, or + `lein clean-ns` will complain). +2. Replace the engine's data-map entry with: + ```clojure + :the-engine {::protocol-engine? true + ::instance-constructor /-> + ::eval-key-channel } + ``` +3. Do **not** delete the shared failure delays. + +### PITFALL 4 — do not delete shared `*-failure` delays + +`core-async-failure` is used by `:core-async.iterative-scheduling`, +`:applicative.core-async`, and the `>channel-leaf` macro. The other `*-failure` +delays are likewise shared. Migrating one engine does not free its delay. Each +migrated engine gets its **own** `enable-deref` in its facade namespace; leave +the `v0.clj` delays alone until every engine that uses one is migrated. + +### PITFALL 5 — keep ONE `let` level in `eval` / `eval-key` / `eval-key-channel` + +The dispatch functions already have the right shape. Do not add a nested `let`. +The instance is bound once, guarded by `when`: + +```clojure +(let [engine-data (engine-data engine-name) + protocol-engine? (::protocol-engine? engine-data) + engine (when protocol-engine? (protocol-engine engine-name engine-data))] + (if protocol-engine? + (engine.protocols/eval engine env k (engine.protocols/-prepare-opts engine opts)) + (let [efn (engine-fn engine-name 'eval)] + (if-let [opts ((::opts-fn engine-data) opts)] + (efn env k opts) + (efn env k))))) +``` + +If you already migrated an engine, these three functions need NO further change +— they are generic. + +--- + +## Editing the tests + +### PITFALL 6 — the graceful-degradation test must target the engine's OWN delay + +`test/nodely/api_test.clj` simulates "library missing" by bombing `require` and +resetting a delay. A migrated engine no longer consults the `v0.clj` +`*-failure` delay — it consults its **own** `enable-deref` in its facade +namespace. So for the engine you migrate: + +- point the `testing-require-delay` block's bombed namespace at the engine's + real implementation namespace (the one its `enable-deref` requires), and +- point the reset delay at `/enable-deref`. + +The reset helpers (`ensure-unrealized-delay` and the end-of-block reload) already +reload the delay's **own** namespace via `(symbol (namespace sym))`, so passing a +facade-namespace delay works without further change. Keep the assertion that the +engine throws its "Could not locate ..." message. + +--- + +## PITFALL 7 — the applicative-family engines are NOT simple; do them last + +`:applicative.promesa`, `:applicative.core-async`, and +`:applicative.virtual-future` all share the implementation namespace +`nodely.engine.applicative`, which **itself requires `clojure.core.async`** and +injects a per-engine "context" resolved from an optional namespace. That means +the facade-must-not-transitively-require-the-optional-dep rule is harder to +satisfy, and `-prepare-opts` must reproduce the context injection. Do the +standalone engines first (`:core-async.iterative-scheduling`, `:async.manifold`, +`:async.virtual-futures`). Ask a human before attempting the applicative family. + +--- + +## Verify (PITFALL 8 — do all of these, every time) + +Run these after each engine. Do not skip. + +1. **Parse** every file you touched: + `bb -e '(require (quote [rewrite-clj.zip :as z])) (z/of-file "PATH")'` +2. **Format / namespaces:** `lein format` and `lein clean-ns` (both dry) — must + report nothing to change. +3. **Tests:** `lein test`. Expect `0 failures, 0 errors`. + - The one test in `nodely.engine.manifold-test` that compares timing is + **flaky** (it allows only an 8ms tolerance on a ~2-second measurement). If + it — and only it — fails on timing, just run `lein test` again. Any other + failure is a real regression. +4. Re-run the touched namespaces a couple of times + (`lein test nodely.api-test nodely.profile-test`) to confirm the test + reset logic is stable. + +Do not commit. Leave commits to a human reviewer. diff --git a/src/nodely/api/v0.clj b/src/nodely/api/v0.clj index 2f9aeeb..f23cf8d 100644 --- a/src/nodely/api/v0.clj +++ b/src/nodely/api/v0.clj @@ -4,6 +4,7 @@ [nodely.data] [nodely.engine.applicative :as applicative] [nodely.engine.core :as engine-core] + [nodely.engine.core-async.lazy-scheduling-engine :as engine.core-async.lazy-scheduling-engine] [nodely.engine.lazy :as engine.lazy] [nodely.engine.protocols :as engine.protocols] [nodely.syntax :as syntax] @@ -73,9 +74,8 @@ :cause e})))) (def engine-data - {:core-async.lazy-scheduling {::ns-name 'nodely.engine.core-async.lazy-scheduling - ::opts-fn identity - ::enable-deref core-async-failure + {:core-async.lazy-scheduling {::protocol-engine? true + ::instance-constructor engine.core-async.lazy-scheduling-engine/->CoreAsyncLazySchedulingEngine ::eval-key-channel true} :core-async.iterative-scheduling {::ns-name 'nodely.engine.core-async.iterative-scheduling ::opts-fn identity @@ -130,6 +130,20 @@ (def engine-fn (memoize data-engine-function)) +(defn- protocol-engine + "Instantiates the protocol engine registered under `engine-name` and, via its + `-enable-deref`, verifies it can run on the current classpath -- throwing an + informative error otherwise. Returns the ready-to-use engine instance." + [engine-name engine-data] + (let [engine ((::instance-constructor engine-data))] + (when-let [{:keys [msg cause] :as enable-failure} @(engine.protocols/-enable-deref engine)] + (throw (ex-info msg + (-> enable-failure + (dissoc :msg :cause) + (assoc ::specified-engine-name engine-name)) + cause))) + engine)) + (defn eval ([env k] (eval env k {})) @@ -138,7 +152,7 @@ :as opts}] (let [engine-data (engine-data engine-name) protocol-engine? (::protocol-engine? engine-data) - engine (when protocol-engine? ((::instance-constructor engine-data)))] + engine (when protocol-engine? (protocol-engine engine-name engine-data))] (if protocol-engine? (engine.protocols/eval engine env k (engine.protocols/-prepare-opts engine opts)) (let [efn (engine-fn engine-name 'eval)] @@ -154,7 +168,7 @@ :as opts}] (let [engine-data (engine-data engine-name) protocol-engine? (::protocol-engine? engine-data) - engine (when protocol-engine? ((::instance-constructor engine-data)))] + engine (when protocol-engine? (protocol-engine engine-name engine-data))] (if protocol-engine? (engine.protocols/eval-key engine env k (engine.protocols/-prepare-opts engine opts)) (let [efn (engine-fn engine-name 'eval-key)] @@ -166,11 +180,11 @@ ([env k] (eval-key-channel env k {})) ([env k {engine-name ::engine - :or {engine-name :core-async.lazy-scheduling} - :as opts}] + :or {engine-name :core-async.lazy-scheduling} + :as opts}] (let [engine-data (engine-data engine-name) protocol-engine? (::protocol-engine? engine-data) - engine (when protocol-engine? ((::instance-constructor engine-data)))] + engine (when protocol-engine? (protocol-engine engine-name engine-data))] (if protocol-engine? (engine.protocols/eval-key-channel engine env k (engine.protocols/-prepare-opts engine opts)) (let [efn (engine-fn engine-name 'eval-key-channel)] diff --git a/src/nodely/engine/core_async/lazy_scheduling_engine.clj b/src/nodely/engine/core_async/lazy_scheduling_engine.clj new file mode 100644 index 0000000..61d85a5 --- /dev/null +++ b/src/nodely/engine/core_async/lazy_scheduling_engine.clj @@ -0,0 +1,52 @@ +(ns nodely.engine.core-async.lazy-scheduling-engine + (:require + [nodely.engine.protocols :as engine.protocols])) + +;; This namespace is intentionally free of any compile-time dependency on +;; `clojure.core.async` so that `CoreAsyncLazySchedulingEngine` can be +;; constructed -- and asked, via `-enable-deref`, whether it can run -- even +;; when core.async is absent from the classpath. The engine does no core.async +;; work until an `-eval*` method is called; the implementation namespace (which +;; does require core.async) is loaded lazily at that point, and by +;; `-enable-deref`. + +(def ^:private impl-ns 'nodely.engine.core-async.lazy-scheduling) + +(def enable-deref + "A delay yielding nil when the core.async lazy-scheduling implementation can + be loaded (i.e. core.async is on the classpath), or a failure map describing + the missing dependency otherwise." + (delay + (try + (require impl-ns) + nil + (catch Exception e + {:msg "Could not locate core-async on classpath." + ::error :missing-ns + ::requested-namespaces [impl-ns] + :cause e})))) + +(defn- impl + "Lazily loads the implementation namespace and resolves `fn-name` within it." + [fn-name] + (requiring-resolve (symbol (name impl-ns) (name fn-name)))) + +(deftype CoreAsyncLazySchedulingEngine [] + engine.protocols/Engine + (-eval [_engine env k opts] + ((impl 'eval) env k opts)) + + (-eval-key [_engine env k opts] + ((impl 'eval-key) env k opts)) + + (-eval-key-channel [_engine env k opts] + ((impl 'eval-key-channel) env k opts)) + + (-eval-key-channel-supported? [_engine] + true) + + (-enable-deref [_engine] + enable-deref) + + (-prepare-opts [_engine opts] + opts)) diff --git a/test/nodely/api_test.clj b/test/nodely/api_test.clj index 7cccda0..2e61eb7 100644 --- a/test/nodely/api_test.clj +++ b/test/nodely/api_test.clj @@ -62,7 +62,7 @@ (defn ensure-unrealized-delay [sym] (when (realized? (deref (resolve sym))) - (require :reload '[nodely.api.v0]))) + (require :reload (symbol (namespace sym))))) (defn- testing-require-delay-call [ns-sym delay message cause call-fn] @@ -75,7 +75,7 @@ (apply orig-require args))) res (with-redefs [require bomb] (call-fn))] - (require :reload 'nodely.api.v0) + (require :reload (symbol (namespace delay))) res))) (defmacro testing-require-delay @@ -100,7 +100,7 @@ 5 (async/