Redesign of Julia interop, avoiding Jupyter kernel

.gitignore

@@ -109,3 +109,6 @@ catcolab-vm.qcow2
 # Markdown previews
 README.html
 CHANGELOG.html
+
+# VSCode
+.vscode/*

packages/algjulia-interop/Project.toml

@@ -1,53 +1,28 @@
 name = "CatColabInterop"
 uuid = "9ecda8fb-39ab-46a2-a496-7285fa6368c1"
 license = "MIT"
-authors = ["CatColab team"]
 version = "0.1.1"
+authors = ["CatColab team"]
 
 [deps]
-ACSets = "227ef7b5-1206-438b-ac65-934d6da304b8"
-Catlab = "134e5e36-593f-5add-ad60-77f754baafbe"
-CombinatorialSpaces = "b1c52339-7909-45ad-8b6a-6e388f7c67f2"
-ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
-CoordRefSystems = "b46f11dc-f210-4604-bfba-323c1ec968cb"
-Decapodes = "679ab3ea-c928-4fe6-8d59-fd451142d391"
-DiagrammaticEquations = "6f00c28b-6bed-4403-80fa-30e0dc12f317"
-Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
-GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
-IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
-LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 MLStyle = "d8e11817-5142-5d16-987a-aa16d5891078"
-OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
-Preferences = "21216c6a-2e73-6563-6e65-726566657250"
-REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 
 [weakdeps]
-PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
+ACSets = "227ef7b5-1206-438b-ac65-934d6da304b8"
+Catlab = "134e5e36-593f-5add-ad60-77f754baafbe"
 
 [extensions]
-SysImageExt = "PackageCompiler"
+CatlabExt = ["Catlab", "ACSets"]
 
 [compat]
-ACSets = "0.2.21"
-Catlab = "0.16.20"
-CombinatorialSpaces = "0.7.4"
-ComponentArrays = "0.15"
-CoordRefSystems = "0.18.9"
-Decapodes = "0.6"
-DiagrammaticEquations = "0.2"
-Distributions = "0.25"
-GeometryBasics = "0.5.7"
-IJulia = "1.26.0"
-JSON3 = "1"
-LinearAlgebra = "1"
-MLStyle = "0.4"
-OrdinaryDiffEq = "6.101.0"
-PackageCompiler = "2.2.1"
-Preferences = "1.5.0"
-REPL = "1.11.0"
+Catlab = "0.17.2"
+HTTP = "1.10.19"
+MLStyle = "0.4.17"
+Oxygen = "1.7.5"
 Reexport = "1.2.2"
-StaticArrays = "1"
+StructTypes = "1.11.0"
 julia = "1.11"

packages/algjulia-interop/README.md

@@ -1,47 +1,39 @@
 # AlgebraicJulia Service
 
 This small package makes functionality from
-[AlgebraicJulia](https://www.algebraicjulia.org/) available to CatColab,
-intermediated by a Julia kernel running in the [Jupyter](https://jupyter.org/)
-server. At this time, only a
-[Decapodes.jl](https://github.com/AlgebraicJulia/Decapodes.jl) service is
-provided. Other packages may be added in the future.
-
-## Setup
-
-1. Install [Julia](https://julialang.org/), say by using
-[`juliaup`](https://github.com/JuliaLang/juliaup)
-2. Install [Jupyter](https://jupyter.org/), say by using `pip` or `conda`
-3. Install [IJulia](https://github.com/JuliaLang/IJulia.jl), which provides the
-   Julia kernel to Jupyter
-
-At this stage, you should be able to launch a Julia kernel inside a JupyterLab.
-
-Having done that, navigate to this directory and run:
-
-```sh
-julia --project -e 'import Pkg; Pkg.instantiate()'
-```
+[AlgebraicJulia](https://www.algebraicjulia.org/) available to CatColab. At this 
+time, only a [Catlab.jl](https://github.com/AlgebraicJulia/Catlab.jl) service is
+provided. Other packages (e.g. Decapodes.jl) will be added in the future.
 
 ## Usage
 
-To start the server, run the following in a Julia REPL
-```julia
-using CatColabInterop
-start_server!()
-```
-This will run the Jupyter kernel in the REPL. You may stop the server by
-running `stop_server!()`. While the Jupyter server is running, the AlgebraicJulia service will be usable by CatColab when served locally.
-
-## Compiling a Sysimage
+First, install [Julia](https://julialang.org/), say by using
+[`juliaup`](https://github.com/JuliaLang/juliaup).
+
+We then need a Julia environment that has all the requisite packages installed.
+The `test` folder of this repo is a perfectly good candidate for this, although
+in principal one might want to use their own environment (if they don't want to
+load all dependencies for all possible analyses, or if they have a locally
+modified version of the code that is running the analysis). To make sure this
+environment is ready to use, navigate to this directory and run `julia --project=test` 
+and then press `]` (to enter package mode) and enter the
+commands `instantiate` and `precompile`. If the source code of
+CatColabInterop.jl is different from the latest tagged release, one must also
+run `dev .` from package mode.
+
+Having done that, to start the server, from this directory run:
 
-Precompiling dependencies like `CairoMakie.jl` and `OrdinaryDiffEq.jl` can be
-time-consuming. A **sysimage** is a file that stores precompilation statements,
-making future invocations of `AlgebraicJuliaService` and its dependencies
-immediate.
+```sh
+julia --project=test scripts/endpoint.jl
+```
 
-To build a sysimage, run `build_sysimage()` in a REPL where `CatColabInterop`
-module is in scope. This process may take upwards of five minutes or longer, depending on your machine.
+This starts an instance that is listening on localhost port 8080. When you run 
+CatColab, you should be able to use analyses that communicate with Julia via 
+this address.
 
-Building a sysimage installs an additional kernel which points to the sysimage. You may change the kernel to your sysimage by running `change_kernel!()`, which will populate a menu of kernels in your IJulia kernel directory.
+## For developers
 
+If one is interested in using a version of CatColabInterop.jl that doesn't match
+the latest tagged release, then one must first open the Julia environment from the 
+test directory (`julia --project=test`) and declare one wants to use the local 
+version of the package (press `]` and then `dev .`)

packages/algjulia-interop/ext/CatlabExt.jl

@@ -0,0 +1,120 @@
+module CatlabExt
+
+using ACSets
+using Catlab: Presentation, FreeSchema, Left
+import Catlab: id, dom
+using Catlab.CategoricalAlgebra.Pointwise.FunctorialDataMigrations.Yoneda: 
+  yoneda, colimit_representables, DiagramData
+using CatColabInterop, Oxygen, HTTP
+import CatColabInterop: endpoint
+
+""" 
+Take a parsed CatColab model of ThSchema and make a Catlab schema. Also 
+collect the mapping from UUIDs to human-readable names. 
+"""
+function model_to_schema(m::Model)::Tuple{Schema, Dict{String,Symbol}}
+  obs, homs, attrtypes, attrs = Symbol[],[],[],[]
+  names = Dict{String, Symbol}()
+
+  for stmt in m.obGenerators
+    names[stmt.id] = Symbol(only(stmt.label))
+    if stmt.obType.content == "Entity"
+        push!(obs, names[stmt.id])
+    elseif stmt.obType.content == "AttrType"
+        push!(attrtypes, names[stmt.id])
+    else 
+      error(stmt.obType)
+    end
+  end
+
+  for stmt in m.morGenerators
+    h = (Symbol(only(stmt.label)), names[stmt.dom.content], 
+         names[stmt.cod.content])
+    names[stmt.id] = h[1]
+    if stmt.morType.content == "Attr"
+      push!(attrs, h)
+    else 
+      push!(homs, h)
+    end
+  end
+  (Schema(Presentation(BasicSchema{Symbol}(obs, homs, attrtypes, attrs, []))), 
+   names)
+end
+
+""" 
+Take a CatColab diagram in a model of ThSchema and construct the input data 
+that gets parsed normally from `@acset_colim`. Mutate an existing mapping of 
+UUIDs to schema-level names to include UUID mappings for instance-level names.
+"""
+function diagram_to_data(d::Types.Diagram, names::Dict{String,Symbol}
+                        )::DiagramData
+  data = DiagramData()
+  for o in d.obGenerators
+    names[o.id] = Symbol(only(o.label))
+    push!(data.reprs[names[o.over.content]], names[o.id])
+  end
+  for m in d.morGenerators
+    p1 = names[m.cod.content] => Symbol[]
+    p2 = names[m.dom.content] => [names[m.over.content]]
+    push!(data.eqs, p1 => p2)
+  end
+  data
+end
+
+""" 
+Receiver of the data already knows the schema, so the JSON payload to CatColab
+just includes the columns of data. Every part is named, so we use the names 
+(including for primary key columns) rather than numeric indices.
+"""
+function acset_to_json(X::ACSet, S::Schema, ids::Dict{String, Symbol}, names::Dict{Symbol, Vector{String}}
+                      )::AbstractDict
+  Dict{String, Vector{String}}(
+    [findfirst(==(t), ids) => names[t] for t in types(S)] 
+    ∪ [findfirst(==(f), ids) => names[c][X[f]] for (f,_,c) in homs(S)] 
+    ∪ [findfirst(==(f), ids) => names[c][getvalue.(X[f])] for (f,_,c) in attrs(S)] )
+end
+
+"""
+Pick a human-readable name for all parts of the ACSet, given explicit names for 
+some of the parts. There is some ambiguity here (the vertex of a generic 
+reflexive edge `e` could be either `src(e)` or `tgt(e)`), but an arbitrary name
+is chosen after minimizing length (`src(e)` preferred over `src(refl(src(e)))`). 
+"""
+function make_names(res::ACSet, names::NamedTuple
+                   )::Dict{Symbol, Vector{String}}
+  S = acset_schema(res)
+  function get_name(o::Symbol, i, curr=[])::Vector{Vector{Symbol}}
+    V(x) = o in attrtypes(S) ? AttrVar(x) : x # embellish attrvars
+    L(x) = o in attrtypes(S) ? Left(x) : x    # embellish attrvars
+    found = findfirst(==((o, L(i))), names)     # if (o,i) is in names
+    isnothing(found) || return [[found; curr]]  # then just give the name
+    inc = [(d, new_i, f) for (f, d, _) in arrows(S, to=o) 
+           for new_i in incident(res, V(i), f)]
+    return vcat([get_name(d, new_i, [f; curr]) for (d, new_i, f) in inc]...)
+  end
+  return Dict{Symbol, Vector{String}}(map(types(acset_schema(res))) do o
+    o => map(parts(res, o)) do i
+      possible_names = sort(get_name(o, i); by=length)
+      foldl((x,y)->"$y($x)", string.(first(possible_names)))
+    end
+  end)
+end
+
+""" 
+Top level function called by CatColab. Computes an ACSet colimit of a 
+  diagrammatic instance. Return a JSON tabular representation.
+"""
+function endpoint(::Val{:ACSetColim})
+  @post "/acsetcolim" function(req::HTTP.Request)
+    payload = json(req, ModelDiagram)
+    schema, ids = model_to_schema(payload.model)
+    data = diagram_to_data(payload.diagram, ids)
+    acset_type = AnonACSet(
+      schema; type_assignment=Dict(a=>Nothing for a in schema.attrtypes))
+    y = yoneda(constructor(acset_type))
+    names, res = colimit_representables(data, y)
+    acset_to_json(res, schema, ids, make_names(res, names))
+  end
+end
+
+end # module

packages/algjulia-interop/scripts/endpoint.jl

@@ -0,0 +1,50 @@
+
+# Example usage:
+
+# julia --project=my_alg_julia_env --threads 4 endpoint.jl Catlab AlgebraicPetri
+
+# Where my_alg_julia_env is a Julia environment with CatColabInterop, Oxygen, 
+# HTTP, and any AlgJulia dependencies.
+
+using CatColabInterop
+using Oxygen
+using HTTP
+
+const CORS_HEADERS = [
+    "Access-Control-Allow-Origin" => "*",
+    "Access-Control-Allow-Headers" => "*",
+    "Access-Control-Allow-Methods" => "POST, GET, OPTIONS"
+]
+
+function CorsHandler(handle)
+    return function (req::HTTP.Request)
+        # return headers on OPTIONS request
+        if HTTP.method(req) == "OPTIONS"
+            return HTTP.Response(200, CORS_HEADERS)
+        else
+            r = handle(req)
+            r.headers = CORS_HEADERS
+            r
+        end
+    end
+end
+
+defaults = [:Catlab,:ACSets] # all extensions to date
+
+# Dynamically load packages in command lin eargs
+for pkg in (isempty(ARGS) ? defaults : ARGS )
+  @info "using $pkg"
+  @eval using $pkg
+end
+
+for m in methods(CatColabInterop.endpoint)
+  sig = m.sig.parameters
+  (length(sig)==2 && sig[2].instance isa Val) || error("Unexpected signature $sig")
+  name = only(sig[2].parameters)
+  @info "Loading endpoint $name"
+  name isa Symbol || error("Unexpected endpoint name $name")
+  fntype, argtypes... = m.sig.types
+  invoke(fntype.instance, Tuple{argtypes...}, Val(name))
+end
+
+serve(middleware=[CorsHandler])