Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
d6260b3
Add Performance workflow benchmarking OpenDJ vs OpenLDAP
vharseko Jun 22, 2026
75c17bf
Rename benchmark workflow and fix OpenLDAP version-capture SIGPIPE
vharseko Jun 22, 2026
f698631
Escape commas in __P(basedn) default to fix JMeter plan compile
vharseko Jun 22, 2026
ff93f3b
Refine LDAP benchmark: READD, SSHA-256 hashing, grouped charts
vharseko Jun 22, 2026
9a666b7
Fix benchmark charts: non-overlapping columns and total throughput
vharseko Jun 22, 2026
3c9705f
Fix benchmark charts: non-overlapping columns and total throughput
vharseko Jun 22, 2026
82016da
Benchmark: search on indexed mail, unique entries, QuickChart charts
vharseko Jun 22, 2026
af83007
Benchmark: lighter default load, log-scale latency, per-server log ar…
vharseko Jun 22, 2026
736c83d
Benchmark: 200 threads default, p99 latency chart (linear)
vharseko Jun 22, 2026
de35266
Benchmark: migrate OpenLDAP to vegardit (2.6), hash with {SSHA}
vharseko Jun 22, 2026
d7a519f
Benchmark: fix OpenLDAP image name to vegardit/openldap
vharseko Jun 22, 2026
5ec5753
Benchmark: make summary.sh reusable, move notes into the workflow
vharseko Jun 23, 2026
2a450ca
build.yml: benchmark the built image against the released one
vharseko Jun 23, 2026
b4b927f
Switch opendj-docker base image to Ubuntu Noble and expand build plat…
vharseko Jun 23, 2026
0b2babe
Fix alpine multi-arch Docker build (conditional JDK)
vharseko Jun 23, 2026
79e1f83
Merge branch 'features/performance-build' into features/docker-noble
vharseko Jun 24, 2026
108ee92
Benchmark: label throughput as tests/s and flip the throughput chart
vharseko Jun 24, 2026
05fb378
Merge branch 'features/performance' into features/performance-build
vharseko Jun 24, 2026
5501d54
Merge branch 'features/performance-build' into features/docker-noble
vharseko Jun 24, 2026
8b0cb43
Benchmark: capture errors/artifacts in build.yml, revert throughput c…
vharseko Jun 24, 2026
bf6e805
Merge branch 'features/performance-build' into features/docker-noble
vharseko Jun 24, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
327 changes: 327 additions & 0 deletions .github/benchmark/benchmark.jmx

Large diffs are not rendered by default.

110 changes: 110 additions & 0 deletions .github/benchmark/compare-opendj.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
# The contents of this file are subject to the terms of the Common Development and
# Distribution License (the License). You may not use this file except in compliance with the
# License.
#
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
# specific language governing permission and limitations under the License.
#
# When distributing Covered Software, include this CDDL Header Notice in each file and include
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
# Header, with the fields enclosed by brackets [] replaced by your own identifying
# information: "Portions copyright [year] [name of copyright owner]".
#
# Copyright 2026 3A Systems, LLC.
#
# Compare two OpenDJ Docker images with the LDAP benchmark (.github/benchmark/benchmark.jmx) and
# append the comparison report to $GITHUB_STEP_SUMMARY. Both sides are OpenDJ, so no per-server
# hashing/index setup is needed (identical product => identical default password scheme).
#
# Usage:
# compare-opendj.sh <A_name> <A_image> <B_name> <B_image>
# Env: THREADS (default 200), DURATION (default 150), JMETER_VERSION (default 5.6.3).
set -euo pipefail

A_NAME="${1:?server A name required}"
A_IMAGE="${2:?server A image required}"
B_NAME="${3:?server B name required}"
B_IMAGE="${4:?server B image required}"

THREADS="${THREADS:-200}"
DURATION="${DURATION:-150}"
JMETER_VERSION="${JMETER_VERSION:-5.6.3}"
BASEDN="dc=example,dc=com"
BENCHPW="benchPass1"
HERE="$(cd "$(dirname "$0")" && pwd)" # .github/benchmark

# ---------------------------------------------------------------- dependencies
if ! command -v ldapsearch >/dev/null 2>&1 || ! command -v jq >/dev/null 2>&1; then
sudo apt-get update -qq
sudo apt-get install -y -qq ldap-utils jq
fi
JM="$HOME/jmeter/apache-jmeter-$JMETER_VERSION/bin/jmeter"
if [ ! -x "$JM" ]; then
mkdir -p "$HOME/jmeter"
curl -fsSL "https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz" -o /tmp/jmeter.tgz
tar -xzf /tmp/jmeter.tgz -C "$HOME/jmeter"
fi

wait_dj() { # poll OpenDJ readiness on localhost:1389
for _ in $(seq 1 90); do
ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \
-b "$BASEDN" -s base dn >/dev/null 2>&1 && return 0
sleep 2
done
return 1
}

# bench_one <image> <out-slug> -> prints the captured server version (stdout only)
bench_one() {
local image="$1" out="$2" ver=""
docker rm -f opendj-bench >/dev/null 2>&1 || true
if docker run -d --name opendj-bench -p 1389:1389 \
-e ROOT_PASSWORD=password -e BASE_DN="$BASEDN" -e ADD_BASE_ENTRY=--addBaseEntry \
"$image" >/dev/null 2>&1; then
wait_dj || echo "WARN: $image not ready in time" >&2
ldapadd -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \
-f "$HERE/people.ldif" >/dev/null 2>&1 || true
ver="$( { ldapsearch -x -LLL -H ldap://localhost:1389 -D 'cn=Directory Manager' -w password \
-b '' -s base fullVendorVersion 2>/dev/null || true; } | sed -n 's/^fullVendorVersion: //p')"
rm -rf "$out" "$out.jtl"
HEAP="-Xms1g -Xmx2g" "$JM" -n -t "$HERE/benchmark.jmx" \
-Jhost=localhost -Jport=1389 -Jbasedn="$BASEDN" \
-Jadminbinddn="cn=Directory Manager" -Jadminbindpw=password -Jbenchpw="$BENCHPW" \
-Jthreads="$THREADS" -Jduration="$DURATION" -Jrampup=0 \
-Jjmeter.reportgenerator.sample_filter='^(?!ADMIN_CONNECT).*' \
-l "$out.jtl" -e -o "$out" > "$out.jmeter.out" 2>&1 || true
docker logs opendj-bench > "$out.docker.log" 2>&1 || true
# surface distinct error messages to the step log (stderr; stdout carries the version)
if [ -f "$out.jtl" ]; then
local errs
errs="$(awk -F',' 'NR==1{for(i=1;i<=NF;i++)h[$i]=i; next}
tolower($h["success"])=="false"{print $h["label"]" | "$h["responseCode"]" | "$h["responseMessage"]}' \
"$out.jtl" 2>/dev/null | sort | uniq -c | sort -rn | head -10)"
[ -z "$errs" ] || { echo "[$out] errors (count | op | code | message):" >&2; echo "$errs" >&2; }
fi
else
echo "ERROR: failed to start image $image" >&2
fi
docker rm -f opendj-bench >/dev/null 2>&1 || true
[ -n "$ver" ] || ver="$image"
printf '%s' "$ver"
}

echo "Benchmarking ${A_NAME} (${A_IMAGE}) @ ${THREADS} threads / ${DURATION}s ..."
A_VER="$(bench_one "$A_IMAGE" a)"
echo "Benchmarking ${B_NAME} (${B_IMAGE}) @ ${THREADS} threads / ${DURATION}s ..."
B_VER="$(bench_one "$B_IMAGE" b)"

{
bash "$HERE/summary.sh" \
"$A_NAME" a/statistics.json "$A_VER" "$A_IMAGE" \
"$B_NAME" b/statistics.json "$B_VER" "$B_IMAGE"
echo ""
echo "### Notes"
echo ""
echo "- **${A_NAME}** = freshly built image; **${B_NAME}** = latest released image. Both are"
echo " OpenDJ, so they share the same default password storage scheme (hashing parity is automatic)."
echo "- The admin connection bind (\`ADMIN_CONNECT\`) is cached per thread and excluded; \`BIND\` is"
echo " the measured user authentication (\`test=sbind\`, single bind/unbind)."
} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
17 changes: 17 additions & 0 deletions .github/benchmark/people.ldif
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# The contents of this file are subject to the terms of the Common Development and
# Distribution License (the License). You may not use this file except in compliance with the
# License.
#
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
# specific language governing permission and limitations under the License.
#
# When distributing Covered Software, include this CDDL Header Notice in each file and include
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
# Header, with the fields enclosed by brackets [] replaced by your own identifying
# information: "Portions copyright [year] [name of copyright owner]".
#
# Copyright 2026 3A Systems, LLC.
dn: ou=People,dc=example,dc=com
objectClass: top
objectClass: organizationalUnit
ou: People
129 changes: 129 additions & 0 deletions .github/benchmark/summary.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
#!/usr/bin/env bash
# The contents of this file are subject to the terms of the Common Development and
# Distribution License (the License). You may not use this file except in compliance with the
# License.
#
# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
# specific language governing permission and limitations under the License.
#
# When distributing Covered Software, include this CDDL Header Notice in each file and include
# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
# Header, with the fields enclosed by brackets [] replaced by your own identifying
# information: "Portions copyright [year] [name of copyright owner]".
#
# Copyright 2026 3A Systems, LLC.
#
# Render an LDAP benchmark comparison report (versions + per-operation table + QuickChart charts)
# for two servers A and B to stdout — intended to be appended to $GITHUB_STEP_SUMMARY. The report
# is generic: server names, versions and images are all parameters, and benchmark-specific notes
# are NOT included here (append them separately, e.g. `cat notes.md >> $GITHUB_STEP_SUMMARY`).
#
# Usage:
# summary.sh <A_name> <A_statistics.json> <A_version> <A_image> \
# <B_name> <B_statistics.json> <B_version> <B_image>
#
# The admin connection bind is labelled ADMIN_CONNECT in the plan and is intentionally
# skipped here, so it never pollutes the per-operation comparison.
set -euo pipefail

A_NAME="${1:?server A name required}"
A_JSON="${2:?server A statistics.json required}"
A_VER="${3:-unknown}"
A_IMG="${4:-}"
B_NAME="${5:?server B name required}"
B_JSON="${6:?server B statistics.json required}"
B_VER="${7:-unknown}"
B_IMG="${8:-}"

A_COLOR="#4e79a7" # blue = server A
B_COLOR="#f28e2b" # orange = server B

# Operations to compare, in workflow order. ADMIN_CONNECT and Total are excluded.
OPS=("ADD" "SEARCH" "COMPARE" "MODIFY" "BIND" "DELETE" "READD")

# m <file> <label> <field> -> numeric value (0 if absent), rounded to 1 decimal.
m() { jq -r --arg l "$2" --arg f "$3" '((.[$l][$f]) // 0) | (.*10 | round / 10)' "$1"; }
# mi <file> <label> <field> -> integer value (0 if absent).
mi() { jq -r --arg l "$2" --arg f "$3" '((.[$l][$f]) // 0) | round' "$1"; }

echo "## 🔬 Benchmark: ${A_NAME} vs ${B_NAME}"
echo ""

# ---------------------------------------------------------------- Versions
echo "### Versions"
echo ""
echo "| Server | Version | Image |"
echo "|---|---|---|"
echo "| **${A_NAME}** | \`${A_VER}\` | \`${A_IMG:-n/a}\` |"
echo "| **${B_NAME}** | \`${B_VER}\` | \`${B_IMG:-n/a}\` |"
echo ""

# ---------------------------------------------------------------- Totals
a_tot_tp=$(m "$A_JSON" Total throughput)
b_tot_tp=$(m "$B_JSON" Total throughput)
a_tot_n=$(mi "$A_JSON" Total sampleCount)
b_tot_n=$(mi "$B_JSON" Total sampleCount)
a_tot_e=$(mi "$A_JSON" Total errorCount)
b_tot_e=$(mi "$B_JSON" Total errorCount)
a_tot_mean=$(m "$A_JSON" Total meanResTime)
b_tot_mean=$(m "$B_JSON" Total meanResTime)

echo "### Totals (all operations, ADMIN_CONNECT excluded by the plan label)"
echo ""
echo "| Server | Throughput (tests/s) | Mean (ms) | Samples | Errors |"
echo "|---|--:|--:|--:|--:|"
echo "| **${A_NAME}** | ${a_tot_tp} | ${a_tot_mean} | ${a_tot_n} | ${a_tot_e} |"
echo "| **${B_NAME}** | ${b_tot_tp} | ${b_tot_mean} | ${b_tot_n} | ${b_tot_e} |"
echo ""

# ---------------------------------------------------------------- Per-op table
echo "### Per-operation latency"
echo ""
echo "| Operation | mean ms ${A_NAME} | mean ms ${B_NAME} | p99 ms ${A_NAME} | p99 ms ${B_NAME} | err ${A_NAME} | err ${B_NAME} |"
echo "|---|--:|--:|--:|--:|--:|--:|"
for op in "${OPS[@]}"; do
printf '| %s | %s | %s | %s | %s | %s | %s |\n' \
"$op" \
"$(m "$A_JSON" "$op" meanResTime)" "$(m "$B_JSON" "$op" meanResTime)" \
"$(mi "$A_JSON" "$op" pct3ResTime)" "$(mi "$B_JSON" "$op" pct3ResTime)" \
"$(mi "$A_JSON" "$op" errorCount)" "$(mi "$B_JSON" "$op" errorCount)"
done
echo ""

# ---------------------------------------------------------------- Chart helpers (QuickChart)
# Mermaid xychart-beta can't do grouped bars / a legend and crowds 14 x-labels, so render proper
# grouped bar charts via QuickChart (Chart.js) as images: https://quickchart.io/chart?c=<config>.

# JSON array of the OPS labels: ["ADD","SEARCH",...].
labels_json() {
local out="" op
for op in "${OPS[@]}"; do out+="${out:+,}\"${op}\""; done
printf '[%s]' "$out"
}
# Comma-joined values for all OPS from <file> <field> via the <m> helper.
vals() { # <fn> <file> <field>
local fn="$1" file="$2" field="$3" out="" v
for op in "${OPS[@]}"; do v=$("$fn" "$file" "$op" "$field"); out+="${out:+,}${v}"; done
printf '%s' "$out"
}
urienc() { jq -rn --arg s "$1" '$s|@uri'; } # URL-encode the chart config
qc() { printf 'https://quickchart.io/chart?w=%s&h=%s&c=%s' "$1" "$2" "$(urienc "$3")"; }

# ---------------------------------------------------------------- Total throughput chart
echo "### Total throughput (tests/s, higher is better)"
echo ""
echo "_Per-operation throughput is not charted: every op runs once per loop iteration, so each"
echo "op's throughput just equals the loop rate. The meaningful throughput is the aggregate._"
echo ""
TP_CFG="{\"type\":\"bar\",\"data\":{\"labels\":[\"${A_NAME}\",\"${B_NAME}\"],\"datasets\":[{\"label\":\"tests/s\",\"backgroundColor\":[\"$A_COLOR\",\"$B_COLOR\"],\"data\":[${a_tot_tp},${b_tot_tp}]}]},\"options\":{\"legend\":{\"display\":false},\"title\":{\"display\":true,\"text\":\"Total throughput (tests/s)\"}}}"
echo "![Total throughput (tests/s)]($(qc 500 320 "$TP_CFG"))"
echo ""

# ---------------------------------------------------------------- Latency chart (grouped bars)
echo "### p99 latency per operation (ms, lower is better)"
echo ""
echo "_🟦 ${A_NAME} · 🟧 ${B_NAME} — grouped bars per operation._"
echo ""
LAT_CFG="{\"type\":\"bar\",\"data\":{\"labels\":$(labels_json),\"datasets\":[{\"label\":\"${A_NAME}\",\"backgroundColor\":\"$A_COLOR\",\"data\":[$(vals mi "$A_JSON" pct3ResTime)]},{\"label\":\"${B_NAME}\",\"backgroundColor\":\"$B_COLOR\",\"data\":[$(vals mi "$B_JSON" pct3ResTime)]}]},\"options\":{\"title\":{\"display\":true,\"text\":\"p99 latency per operation (ms)\"}}}"
echo "![p99 latency per operation (ms)]($(qc 900 400 "$LAT_CFG"))"
echo ""
Loading
Loading