Skip to content

Commit 1b9f80a

Browse files
lkdvoslkdvos
committed
update benchmarks
1 parent f68996c commit 1b9f80a

3 files changed

Lines changed: 49 additions & 248 deletions

File tree

‎benchmark/MPSKitBenchmarks/derivatives/AC2_benchmarks.jl‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ benchname(spec::AC2Spec) = dim(spec.mps_virtualspaces[1]), dim(spec.mpo_virtuals
2020

2121
# Benchmarks
2222
# ----------
23-
function MPSKit.AC2_hamiltonian(spec::AC2Spec{S}; T::Type = Float64) where {S}
23+
24+
function prepare_state(spec::AC2Spec{S}; T::Type = Float64) where {S}
2425
GL = randn(T, spec.mps_virtualspaces[1] ⊗ spec.mpo_virtualspaces[1]' ← spec.mps_virtualspaces[1])
2526
GR = randn(T, spec.mps_virtualspaces[3] ⊗ spec.mpo_virtualspaces[3] ← spec.mps_virtualspaces[3])
2627

@@ -55,26 +56,19 @@ function MPSKit.AC2_hamiltonian(spec::AC2Spec{S}; T::Type = Float64) where {S}
5556
GLs, GRs = MPSKit.initialize_environments(psi, H, psi)
5657
envs = MPSKit.InfiniteEnvironments(GLs, GRs)
5758

58-
return MPSKit.AC2_hamiltonian(1, psi, H, psi, envs)
59+
return psi, H, envs
5960
end
6061

6162
function contraction_benchmark(spec::AC2Spec; T::Type = Float64)
63+
psi, H, envs = prepare_state(spec; T)
64+
H_eff = MPSKit.AC2_hamiltonian(1, psi, H, psi, envs)
6265
V = spec.mps_virtualspaces[1] ⊗ spec.physicalspaces[1] ← spec.mps_virtualspaces[3] ⊗ spec.physicalspaces[2]'
63-
H_eff = MPSKit.AC2_hamiltonian(spec; T)
6466
return @benchmarkable $H_eff * x setup = x = randn($T, $V)
6567
end
6668

6769
function preparation_benchmark(spec::AC2Spec; T::Type = Float64)
68-
V = spec.mps_virtualspaces[1] ⊗ spec.physicalspaces[1] ← spec.mps_virtualspaces[3] ⊗ spec.physicalspaces[2]'
69-
H_eff = MPSKit.AC2_hamiltonian(spec; T)
70-
return @benchmarkable MPSKit.prepare_operator!!($H_eff)
71-
end
72-
73-
function prepared_benchmark(spec::AC2Spec; T::Type = Float64)
74-
V = spec.mps_virtualspaces[1] ⊗ spec.physicalspaces[1] ← spec.mps_virtualspaces[3] ⊗ spec.physicalspaces[2]'
75-
H_eff = MPSKit.AC2_hamiltonian(spec; T)
76-
H_prep = MPSKit.prepare_operator!!(H_eff)
77-
return @benchmarkable $H_prep * x setup = x = randn($T, $V)
70+
psi, H, envs = prepare_state(spec; T)
71+
return @benchmarkable MPSKit.AC2_hamiltonian(1, $psi, $H, $psi, $envs)
7872
end
7973

8074
# Converters

‎benchmark/MPSKitBenchmarks/derivatives/DerivativesBenchmarks.jl‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,22 +25,18 @@ T = Float64
2525

2626
suite_init = addgroup!(SUITE, "AC2_preparation")
2727
suite_apply = addgroup!(SUITE, "AC2_contraction")
28-
suite_prepped = addgroup!(SUITE, "AC2_prepared")
2928

3029
for (model, params) in allparams
3130
g_prep = addgroup!(suite_init, model)
32-
g_prepped = addgroup!(suite_prepped, model)
3331
g_contract = addgroup!(suite_apply, model)
3432
for (symmetry, specs) in params
3533
g_prep_sym = addgroup!(g_prep, symmetry)
3634
g_contract_sym = addgroup!(g_contract, symmetry)
37-
g_prepped_sym = addgroup!(g_prepped, symmetry)
3835
for spec_dict in specs
3936
spec = untomlify(AC2Spec, spec_dict)
4037
name = benchname(spec)
4138
g_contract_sym[name] = contraction_benchmark(spec; T)
4239
g_prep_sym[name] = preparation_benchmark(spec; T)
43-
g_prepped_sym[name] = prepared_benchmark(spec; T)
4440
end
4541
end
4642
end

‎benchmark/plot_results.jl‎

Lines changed: 42 additions & 231 deletions
Original file line numberDiff line numberDiff line change
@@ -6,39 +6,46 @@ using Statistics
66
# Loading in the data
77
# -------------------
88
resultdir = joinpath(@__DIR__, "results")
9-
resultfile(i) = "results_MPSKit@bench$i.json"
109

11-
versions = [0, 1, 2, 3, 5]
10+
result_files = Dict(
11+
"main" => joinpath(resultdir, "results_MPSKit@main.json"),
12+
"dirty" => joinpath(resultdir, "results_MPSKit@dirty.json")
13+
)
1214

13-
df_contract = let df = DataFrame(
14-
:version => Int[], :model => String[], :symmetry => String[],
15-
:D => Int[], :V => Int[], :memory => Int[], :allocs => Int[], :times => Vector{Int}[]
16-
)
1715

18-
for version in versions
19-
result = JSON.parsefile(joinpath(resultdir, resultfile(version)))
16+
df = let df = DataFrame(
17+
:version => String[], :model => String[], :symmetry => String[],
18+
:D => Int[], :V => Int[], :memory => Tuple{Int, Int}[], :allocs => Tuple{Int, Int}[], :times => Tuple{Vector{Int}, Vector{Int}}[]
19+
)
20+
for (version, result_file) in pairs(result_files)
21+
result = JSON.parsefile(result_file)
2022
for (model, model_res) in result.data.derivatives.data.AC2_contraction.data
2123
for (symmetry, sym_res) in model_res.data
22-
for (DV, bench) in sym_res.data
24+
for (DV, contract_bench) in sym_res.data
25+
prep_bench = result.data.derivatives.data.AC2_preparation.data[model].data[symmetry].data[DV]
2326
D, V = eval(Meta.parse(DV))::Tuple{Int, Int}
24-
2527
push!(
2628
df,
27-
(version, model, symmetry, D, V, bench.memory, bench.allocs, collect(Int, bench.times))
29+
(
30+
version, model, symmetry, D, V,
31+
(prep_bench.memory, contract_bench.memory),
32+
(prep_bench.allocs, contract_bench.allocs),
33+
(collect(Int, prep_bench.times), collect(Int, contract_bench.times)),
34+
)
2835
)
2936
end
3037
end
3138
end
3239
end
3340
df
3441
end
42+
3543
df_prep = let df = DataFrame(
36-
:version => Int[], :model => String[], :symmetry => String[],
44+
:version => String[], :model => String[], :symmetry => String[],
3745
:D => Int[], :V => Int[], :memory => Int[], :allocs => Int[], :times => Vector{Int}[]
3846
)
39-
40-
for version in versions
41-
result = JSON.parsefile(joinpath(resultdir, resultfile(version)))
47+
for (version, result_file) in pairs(result_files)
48+
result = JSON.parsefile(result_file)
4249
for (model, model_res) in result.data.derivatives.data.AC2_preparation.data
4350
for (symmetry, sym_res) in model_res.data
4451
for (DV, bench) in sym_res.data
@@ -60,63 +67,31 @@ end
6067
fontsize = 20
6168
estimator = median
6269

63-
f_times = let f = Figure(; size = (1400, 1400))
64-
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
65-
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
66-
67-
68-
df_model = groupby(df_contract, [:model, :symmetry])
69-
for row in eachindex(models), col in eachindex(symmetries)
70-
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
71-
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "Δt (μs)", yscale = log10)
72-
@assert !isnothing(df_data)
73-
for (k, v) in pairs(groupby(df_data, :version))
74-
Ds = v[!, :D]
75-
times = estimator.(v[!, :times]) ./ 1.0e3
76-
I = sortperm(Ds)
77-
scatterlines!(ax, Ds[I], times[I]; label = "v$(k.version)")
78-
end
79-
axislegend(ax, position = :lt)
80-
end
81-
82-
Label(f[0, 0], "times"; fontsize)
83-
for (row, model) in enumerate(models)
84-
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
85-
end
86-
for (col, symmetry) in enumerate(symmetries)
87-
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
88-
end
89-
90-
f
91-
end
92-
save(joinpath(resultdir, "bench_times.png"), f_times)
93-
94-
f_times_relative = let f = Figure(; size = (1400, 1400))
70+
function plot_result(df, num_applications, choice = :times)
71+
f = Figure(; size = (1400, 1400))
9572
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
9673
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
9774

9875

99-
df_model = groupby(df_contract, [:model, :symmetry])
76+
df_model = groupby(df, [:model, :symmetry])
10077
for row in eachindex(models), col in eachindex(symmetries)
10178
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
102-
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "Δt / Δt₀")
103-
hlines!([1], color = :red)
79+
ylabel_ = choice === :times ? "Δt (μs)" : string(choice)
80+
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = ylabel_, yscale = log10)
10481
@assert !isnothing(df_data)
105-
106-
df_v = groupby(df_data, :version)
107-
108-
v = get(df_v, (; version = 0), nothing)
109-
Ds = v[!, :D]
110-
times = estimator.(v[!, :times])
111-
I = sortperm(Ds)
112-
times₀ = times[I]
113-
11482
for (k, v) in pairs(groupby(df_data, :version))
115-
k.version == 0 && continue
11683
Ds = v[!, :D]
84+
if choice === :times
85+
times_prep = estimator.(first.(v[!, :times])) ./ 1.0e3
86+
times_contract = estimator.(last.(v[!, :times])) ./ 1.0e3
87+
data = times_prep .+ (num_applications .* times_contract)
88+
else
89+
allocs_prep = first.(v[!, choice]) ./ 1.0e3
90+
allocs_contract = last.(v[!, choice]) ./ 1.0e3
91+
data = allocs_prep .+ (num_applications .* allocs_contract)
92+
end
11793
I = sortperm(Ds)
118-
times = estimator.(v[!, :times])[I]
119-
scatterlines!(ax, Ds[I], times ./ times₀; label = "v$(k.version)")
94+
scatterlines!(ax, Ds[I], data[I]; label = "$(k.version)")
12095
end
12196
axislegend(ax, position = :lt)
12297
end
@@ -129,174 +104,10 @@ f_times_relative = let f = Figure(; size = (1400, 1400))
129104
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
130105
end
131106

132-
f
133-
end
134-
save(joinpath(resultdir, "bench_times_relative.png"), f_times_relative)
135-
136-
f_allocs = let f = Figure(; size = (1400, 1400))
137-
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
138-
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
139-
140-
141-
df_model = groupby(df_contract, [:model, :symmetry])
142-
for row in eachindex(models), col in eachindex(symmetries)
143-
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
144-
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "allocs", yscale = log10)
145-
@assert !isnothing(df_data)
146-
for (k, v) in pairs(groupby(df_data, :version))
147-
Ds = v[!, :D]
148-
allocs = estimator.(v[!, :allocs])
149-
I = sortperm(Ds)
150-
scatterlines!(ax, Ds[I], allocs[I]; label = "v$(k.version)")
151-
end
152-
axislegend(ax, position = :lt)
153-
end
154-
155-
Label(f[0, 0], "allocs"; fontsize)
156-
for (row, model) in enumerate(models)
157-
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
158-
end
159-
for (col, symmetry) in enumerate(symmetries)
160-
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
161-
end
162-
163-
f
164-
end
165-
save(joinpath(resultdir, "bench_allocs.png"), f_allocs)
166-
167-
f_memory = let f = Figure(; size = (1400, 1400))
168-
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
169-
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
170-
171-
172-
df_model = groupby(df_contract, [:model, :symmetry])
173-
for row in eachindex(models), col in eachindex(symmetries)
174-
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
175-
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "memory (KiB)", yscale = log10)
176-
@assert !isnothing(df_data)
177-
for (k, v) in pairs(groupby(df_data, :version))
178-
Ds = v[!, :D]
179-
memory = estimator.(v[!, :memory]) ./ (2^10)
180-
I = sortperm(Ds)
181-
scatterlines!(ax, Ds[I], memory[I]; label = "v$(k.version)")
182-
end
183-
axislegend(ax, position = :lt)
184-
end
185-
186-
Label(f[0, 0], "memory"; fontsize)
187-
for (row, model) in enumerate(models)
188-
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
189-
end
190-
for (col, symmetry) in enumerate(symmetries)
191-
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
192-
end
193-
194-
f
195-
end
196-
save(joinpath(resultdir, "bench_memory.png"), f_allocs)
197-
198-
f_memory_relative = let f = Figure(; size = (1400, 1400))
199-
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
200-
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
201-
202-
203-
df_model = groupby(df_contract, [:model, :symmetry])
204-
for row in eachindex(models), col in eachindex(symmetries)
205-
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
206-
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "memory / memory₀")
207-
hlines!([1], color = :red)
208-
@assert !isnothing(df_data)
209-
210-
df_v = groupby(df_data, :version)
211-
212-
v = get(df_v, (; version = 0), nothing)
213-
Ds = v[!, :D]
214-
times = estimator.(v[!, :memory])
215-
I = sortperm(Ds)
216-
times₀ = times[I]
217-
218-
for (k, v) in pairs(groupby(df_data, :version))
219-
k.version == 0 && continue
220-
Ds = v[!, :D]
221-
I = sortperm(Ds)
222-
times = estimator.(v[!, :memory])[I]
223-
scatterlines!(ax, Ds[I], times ./ times₀; label = "v$(k.version)")
224-
end
225-
axislegend(ax, position = :lt)
226-
end
227-
228-
Label(f[0, 0], "memory (relative)"; fontsize)
229-
for (row, model) in enumerate(models)
230-
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
231-
end
232-
for (col, symmetry) in enumerate(symmetries)
233-
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
234-
end
235-
236-
f
107+
return f
237108
end
238-
save(joinpath(resultdir, "bench_memory_relative.png"), f_memory_relative)
239-
240-
241-
# Including preparation times
242-
# ---------------------------
243-
for n_applications in [3, 10, 30]
244-
f_times_relative = let f = Figure(; size = (1400, 1400))
245-
models = ["heisenberg_nn", "heisenberg_nnn", "heisenberg_cylinder", "heisenberg_coulomb"]
246-
symmetries = ["Trivial", "Irrep[U₁]", "Irrep[SU₂]"]
247-
248-
249-
df_model = groupby(df_contract, [:model, :symmetry])
250-
dfp_model = groupby(df_prep, [:model, :symmetry])
251-
for row in eachindex(models), col in eachindex(symmetries)
252-
df_data = get(df_model, (; model = models[row], symmetry = symmetries[col]), nothing)
253-
dfp_data = get(dfp_model, (; model = models[row], symmetry = symmetries[col]), nothing)
254-
ax = Axis(f[row, col], xscale = log10, xlabel = "D", ylabel = "Δt / Δt₀")
255-
hlines!([1], color = :red)
256-
@assert !isnothing(df_data) && !isnothing(dfp_data)
257-
258-
df_v = groupby(df_data, :version)
259-
dfp_v = groupby(dfp_data, :version)
260-
261-
v = get(df_v, (; version = 0), nothing)
262-
Ds = v[!, :D]
263-
times = estimator.(v[!, :times])
264-
I = sortperm(Ds)
265-
times₀ = n_applications .* times[I]
266-
267-
vp = get(dfp_v, (; version = 0), nothing)
268-
Ds = vp[!, :D]
269-
times = estimator.(vp[!, :times])
270-
I = sortperm(Ds)
271-
times₀ .+= times[I]
272-
273-
df_data_v = groupby(dfp_data, :version)
274-
for (k, v) in pairs(groupby(df_data, :version))
275-
k.version == 0 && continue
276-
Ds = v[!, :D]
277-
I = sortperm(Ds)
278-
times = n_applications .* estimator.(v[!, :times])[I]
279-
280-
vp = get(df_data_v, (; k.version), nothing)
281-
@assert !isnothing(vp)
282-
Ds = vp[!, :D]
283-
I = sortperm(Ds)
284-
times .+= estimator.(vp[!, :times][I])
285-
286-
scatterlines!(ax, Ds[I], times ./ times₀; label = "v$(k.version)")
287-
end
288-
axislegend(ax, position = :lt)
289-
end
290-
291-
Label(f[0, 0], "times"; fontsize)
292-
for (row, model) in enumerate(models)
293-
Label(f[row, 0], model; rotation = pi / 2, fontsize, tellheight = false, tellwidth = false)
294-
end
295-
for (col, symmetry) in enumerate(symmetries)
296-
Label(f[0, col], symmetry; fontsize, tellheight = false, tellwidth = false)
297-
end
298-
299-
f
300-
end
301-
save(joinpath(resultdir, "bench_prep_times_relative_n=$n_applications.png"), f_times_relative)
109+
for choice in (:allocs, :memory, :times), n in [1, 3, 10]
110+
f = plot_result(df, n, choice)
111+
save(joinpath(resultdir, "bench_$(choice)_$n.png"), f)
112+
save(joinpath(resultdir, "bench_$(choice)_$n.svg"), f)
302113
end

0 commit comments

Comments
 (0)