test(coverage): hot backup, DB_CONFIG parser, compaction-recovery (env_backup 0->97%, env_config 3->72%)

test/backup/backup_direct.c

@@ -0,0 +1,216 @@
+/*-
+ * See the file LICENSE for redistribution information.
+ *
+ * Copyright (c) 2024 Oracle and/or its affiliates.  All rights reserved.
+ *
+ * backup_direct.c --
+ *	A standalone driver for Berkeley DB's hot-backup configuration and
+ *	callback API (src/env/env_backup.c) and the environment backup engine
+ *	it feeds (src/db/db_backup.c, DB_ENV->backup / DB_ENV->dbbackup).
+ *
+ *	The Tcl harness (test/tcl/backup.tcl) drives hot backup only through
+ *	the db_hotbackup utility, which calls DB_ENV->backup() with a NULL
+ *	backup_handle -- so env_backup.c's four config setters/getters and
+ *	callback setter/getter are never reached, and the backup->open/write/
+ *	close callback branches in db_backup.c stay cold.  This program calls
+ *	those public entry points directly, as an application embedding BDB
+ *	backup would:
+ *
+ *	  __env_get_backup_config (before alloc -> EINVAL branch, then all 4
+ *	      config enums READ_COUNT/READ_SLEEP/SIZE/WRITE_DIRECT),
+ *	  __env_set_backup_config (all 4 enums, WRITE_DIRECT on and off ->
+ *	      F_SET/F_CLR branches, plus __env_backup_alloc first call),
+ *	  __env_get_backup_callbacks (before alloc -> EINVAL branch),
+ *	  __env_set_backup_callbacks (drives __env_backup_alloc's "already
+ *	      allocated" early-return branch), then get again to read them back.
+ *
+ *	It then builds a small transactional environment with a btree db, and
+ *	runs DB_ENV->backup() with the write callbacks installed so the
+ *	backup->open/write/close paths in db_backup.c execute, and finally a
+ *	DB_ENV->dbbackup() single-database backup.  A hard SIGALRM guard aborts
+ *	the process if anything blocks; every handle is closed before exit.
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "db.h"
+
+#define	HOME		"BACKUP_TESTDIR"
+#define	HOME_TGT	"BACKUP_TESTDIR/bak"
+#define	TABLE		"backup_table.db"
+#define	NRECS		200
+#define	ALARM_SECS	60		/* hard self-timeout: never hang */
+
+static int fails = 0;
+
+#define	CHK(call, want) do {						\
+	int _r = (call);						\
+	if (_r != (want)) {						\
+		fprintf(stderr,						\
+		    "FAIL: %s:%d: %s => %d, expected %d\n",		\
+		    __FILE__, __LINE__, #call, _r, (want));		\
+		fails++;						\
+	}								\
+} while (0)
+
+#define	CHK0(call)	CHK((call), 0)
+
+/* Backup callbacks -- a minimal file-copy sink that mirrors the default. */
+static int
+bk_open(DB_ENV *dbenv, const char *dbname, const char *target, void **handle)
+{
+	char path[1024];
+	FILE *fp;
+
+	(void)dbenv;
+	(void)snprintf(path, sizeof(path), "%s/%s", target, dbname);
+	if ((fp = fopen(path, "wb")) == NULL)
+		return (errno == 0 ? EIO : errno);
+	*handle = fp;
+	return (0);
+}
+
+static int
+bk_write(DB_ENV *dbenv, u_int32_t off_gb, u_int32_t off, u_int32_t size,
+    u_int8_t *buf, void *handle)
+{
+	FILE *fp = handle;
+
+	(void)dbenv;
+	(void)off_gb;
+	if (fseek(fp, (long)off, SEEK_SET) != 0)
+		return (EIO);
+	if (fwrite(buf, 1, size, fp) != size)
+		return (EIO);
+	return (0);
+}
+
+static int
+bk_close(DB_ENV *dbenv, const char *dbname, void *handle)
+{
+	FILE *fp = handle;
+
+	(void)dbenv;
+	(void)dbname;
+	if (fp != NULL && fclose(fp) != 0)
+		return (EIO);
+	return (0);
+}
+
+/* Exercise env_backup.c's config + callback API in isolation. */
+static void
+test_config_api(DB_ENV *dbenv)
+{
+	u_int32_t v;
+	int (*op)(DB_ENV *, const char *, const char *, void **);
+	int (*wr)(DB_ENV *, u_int32_t, u_int32_t, u_int32_t, u_int8_t *, void *);
+	int (*cl)(DB_ENV *, const char *, void *);
+
+	/* Getters before any handle exists must fail with EINVAL. */
+	CHK(dbenv->get_backup_config(dbenv, DB_BACKUP_SIZE, &v), EINVAL);
+	CHK(dbenv->get_backup_callbacks(dbenv, &op, &wr, &cl), EINVAL);
+
+	/* Set every config enum -- first set allocates the handle. */
+	CHK0(dbenv->set_backup_config(dbenv, DB_BACKUP_READ_COUNT, 1024));
+	CHK0(dbenv->set_backup_config(dbenv, DB_BACKUP_READ_SLEEP, 500));
+	CHK0(dbenv->set_backup_config(dbenv, DB_BACKUP_SIZE, 1 << 20));
+	/* WRITE_DIRECT on then off -> both F_SET and F_CLR branches. */
+	CHK0(dbenv->set_backup_config(dbenv, DB_BACKUP_WRITE_DIRECT, 1));
+	CHK0(dbenv->set_backup_config(dbenv, DB_BACKUP_WRITE_DIRECT, 0));
+
+	/* Read each one back. */
+	CHK0(dbenv->get_backup_config(dbenv, DB_BACKUP_READ_COUNT, &v));
+	if (v != 1024) { fprintf(stderr, "FAIL: read_count=%u\n", v); fails++; }
+	CHK0(dbenv->get_backup_config(dbenv, DB_BACKUP_READ_SLEEP, &v));
+	if (v != 500) { fprintf(stderr, "FAIL: read_sleep=%u\n", v); fails++; }
+	CHK0(dbenv->get_backup_config(dbenv, DB_BACKUP_SIZE, &v));
+	if (v != (1 << 20)) { fprintf(stderr, "FAIL: size=%u\n", v); fails++; }
+	CHK0(dbenv->get_backup_config(dbenv, DB_BACKUP_WRITE_DIRECT, &v));
+	if (v != 0) { fprintf(stderr, "FAIL: write_direct=%u\n", v); fails++; }
+
+	/* Install callbacks (handle already allocated -> alloc early return). */
+	CHK0(dbenv->set_backup_callbacks(dbenv, bk_open, bk_write, bk_close));
+	CHK0(dbenv->get_backup_callbacks(dbenv, &op, &wr, &cl));
+	if (op != bk_open || wr != bk_write || cl != bk_close) {
+		fprintf(stderr, "FAIL: callbacks not read back\n");
+		fails++;
+	}
+}
+
+int
+main(void)
+{
+	DB_ENV *dbenv;
+	DB *db;
+	DB_TXN *txn;
+	DBT key, data;
+	u_int32_t i;
+	int ret;
+	char kbuf[32], vbuf[64];
+
+	(void)signal(SIGALRM, SIG_DFL);
+	(void)alarm(ALARM_SECS);
+
+	/* Clean start (no rm -rf). */
+	(void)system("rm -f " HOME "/__db.* " HOME "/log.* " HOME "/*.db "
+	    HOME "/DB_CONFIG " HOME_TGT "/* 2>/dev/null");
+	(void)mkdir(HOME, 0755);
+	(void)mkdir(HOME_TGT, 0755);
+
+	if ((ret = db_env_create(&dbenv, 0)) != 0) {
+		fprintf(stderr, "db_env_create: %s\n", db_strerror(ret));
+		return (EXIT_FAILURE);
+	}
+	dbenv->set_errpfx(dbenv, "backup_direct");
+
+	/* Exercise the config/callback API before opening the env. */
+	test_config_api(dbenv);
+
+	CHK0(dbenv->open(dbenv, HOME, DB_CREATE | DB_INIT_LOCK |
+	    DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_RECOVER, 0644));
+
+	CHK0(db_create(&db, dbenv, 0));
+	CHK0(db->open(db, NULL, TABLE, NULL,
+	    DB_BTREE, DB_CREATE | DB_AUTO_COMMIT, 0644));
+
+	CHK0(dbenv->txn_begin(dbenv, NULL, &txn, 0));
+	for (i = 0; i < NRECS; i++) {
+		memset(&key, 0, sizeof(key));
+		memset(&data, 0, sizeof(data));
+		(void)snprintf(kbuf, sizeof(kbuf), "key%08u", i);
+		(void)snprintf(vbuf, sizeof(vbuf), "value-%08u-payload", i);
+		key.data = kbuf; key.size = (u_int32_t)strlen(kbuf) + 1;
+		data.data = vbuf; data.size = (u_int32_t)strlen(vbuf) + 1;
+		CHK0(db->put(db, txn, &key, &data, 0));
+	}
+	CHK0(txn->commit(txn, 0));
+	CHK0(db->close(db, 0));
+
+	/*
+	 * DB_ENV->backup with the write callbacks installed: exercises the
+	 * backup->open/write/close branches in db_backup.c.  DB_CREATE makes
+	 * the target dir, DB_BACKUP_FILES copies db files.
+	 */
+	CHK0(dbenv->backup(dbenv, HOME_TGT,
+	    DB_CREATE | DB_BACKUP_FILES | DB_BACKUP_CLEAN));
+
+	/* Single-database backup -> DB_ENV->dbbackup on one file. */
+	CHK0(dbenv->dbbackup(dbenv, TABLE, HOME_TGT, 0));
+
+	CHK0(dbenv->close(dbenv, 0));
+
+	if (fails != 0) {
+		fprintf(stderr, "backup_direct: %d checks FAILED\n", fails);
+		return (EXIT_FAILURE);
+	}
+	printf("backup_direct: PASS\n");
+	return (EXIT_SUCCESS);
+}

test/backup/run_backup_direct.sh

@@ -0,0 +1,52 @@
+#!/bin/sh -
+#
+# $Id$
+#
+# run_backup_direct.sh --
+#	Build and run the standalone hot-backup API driver (backup_direct.c),
+#	which exercises Berkeley DB's backup config + callback API
+#	(src/env/env_backup.c) and the DB_ENV->backup / DB_ENV->dbbackup engine
+#	(src/db/db_backup.c) via the callback path.  The Tcl backup.tcl test
+#	drives the db_hotbackup utility only, which leaves env_backup.c at 0%
+#	and the backup->open/write/close callback branches of db_backup.c cold.
+#
+# Usage (from build_unix):
+#	sh ../test/backup/run_backup_direct.sh
+#
+# It compiles backup_direct against the just-built libdb in ./.libs, runs it
+# in a guaranteed-clean home under a timeout, and reports PASS/FAIL.  Exits
+# non-zero on failure or hang.
+
+set -e
+
+BUILD=${BUILD:-.}			# build_unix dir (cwd by default)
+SRC=${SRC:-../test/backup/backup_direct.c}
+HOME_DIR=${HOME_DIR:-BACKUP_TESTDIR}
+TIMEOUT=${TIMEOUT:-90}
+
+LIB="$BUILD/.libs/libdb-5.3.so"
+if [ ! -f "$LIB" ]; then
+	# Fall back to the versioned .so actually present.
+	LIB=$(ls "$BUILD"/.libs/libdb-*.so 2>/dev/null | head -1)
+fi
+[ -n "$LIB" ] || { echo "FAIL: libdb .so not found in $BUILD/.libs"; exit 1; }
+
+echo "Compiling backup_direct against $LIB"
+gcc -g -O1 ${CFLAGS:-} -I"$BUILD" "$SRC" "$LIB" \
+    -lpthread -Wl,-rpath,"$(cd "$BUILD/.libs" && pwd)" \
+    -o "$BUILD/backup_direct"
+
+# Guaranteed-clean home: remove env/log/db artifacts (not rm -rf).
+rm -f "$HOME_DIR"/__db.* "$HOME_DIR"/log.* "$HOME_DIR"/*.db \
+    "$HOME_DIR"/DB_CONFIG "$HOME_DIR"/bak/* 2>/dev/null || true
+mkdir -p "$HOME_DIR"/bak
+
+echo "Running backup_direct (timeout ${TIMEOUT}s)"
+if timeout "$TIMEOUT" "$BUILD/backup_direct"; then
+	echo "run_backup_direct.sh: PASS"
+	exit 0
+else
+	rc=$?
+	echo "run_backup_direct.sh: FAIL (rc=$rc)"
+	exit $rc
+fi

test/coverage/README.md

@@ -35,6 +35,7 @@ Knobs (all optional env vars):
 | `COV_TIMEOUT`| `2400`                                    | seconds ceiling for the test run |
 | `COV_REP`    | `0`                                       | set `1` to also run the replication (rep/repmgr) tests |
 | `COV_XA_UPG` | `1`                                       | run the XA + on-disk-upgrade drivers (fast, non-hanging) |
+| `COV_BACKUP` | `1`                                       | run the hot-backup-API + compaction-recovery drivers (fast, non-hanging) |
 | `COV_DEAD_REG` | `1`                                     | run the deadlock-detector + DB_REGISTER multi-process tests |
 
 Set `COV_REP=1` to add the replication suite (biggest cold surface: rep/ +
@@ -110,6 +111,42 @@ timeout, so they cannot hang:
     fixtures here are synthetic, so it is reported to confirm against a genuine
     old fixture, not fixed. See `DB-REPSITE-TODO.md` for the analogous repmgr gap.
 
+## The hot-backup API + compaction-recovery surface (`COV_BACKUP`)
+
+`COV_BACKUP` (on by default) runs two more standalone C drivers, same
+self-clean + hard-timeout shape as the XA/upgrade drivers, aimed at code the
+Tcl suite structurally cannot reach:
+
+- **Hot-backup API** — `test/backup/run_backup_direct.sh` builds
+  `test/backup/backup_direct.c`. `env/env_backup.c` is *only* the four backup
+  config setters/getters (`set/get_backup_config` for `READ_COUNT`/
+  `READ_SLEEP`/`SIZE`/`WRITE_DIRECT`) and the backup-callback setter/getter
+  (`set/get_backup_callbacks`). The Tcl `backup.tcl` test drives hot backup
+  only through the `db_hotbackup` utility, which calls `DB_ENV->backup()` with
+  a **NULL** `backup_handle` — so none of `env_backup.c` runs and the
+  `backup->open/write/close` callback branches of `db/db_backup.c` stay cold.
+  The driver calls those public entry points directly (as an embedding app
+  would): getters-before-alloc (the `EINVAL` branches), all four config enums,
+  `WRITE_DIRECT` on **and** off (both `F_SET`/`F_CLR`), then installs write
+  callbacks and runs `DB_ENV->backup()` + `DB_ENV->dbbackup()` so the callback
+  copy path executes. Lift: `env_backup.c` **0%→~97% line / ~82% branch**,
+  plus `db/db_backup.c`'s callback path (**~43% line / ~30% branch**).
+- **Compaction recovery** — `test/db/run_recd_compact.sh` builds
+  `test/db/recd_compact.c`. Three recovery handlers in `db/db_rec.c` —
+  `__db_merge_recover`, `__db_pgno_recover`, `__db_pg_trunc_recover` (~330
+  lines) — fire only when btree **compaction** / page **truncation** log
+  records are replayed, and no `recd0NN` test runs compaction under recovery,
+  so they were completely cold. The driver fills a small-page btree, deletes a
+  large contiguous range to leave sparse/empty pages, runs
+  `DB->compact(DB_FREE_SPACE)` in a txn (logging `__db_merge`/`__db_pgno`/
+  `__db_pg_trunc` records), then re-opens under `DB_RECOVER_FATAL`
+  (catastrophic recovery replays the whole log from the start → the redo /
+  forward-roll branches of those handlers). It verifies the db still opens and
+  reads back afterward. Combined with `recd002:btree` (splits) + `recd016`
+  (catastrophic recovery), both added to the default `COV_TESTS`, this lifts
+  `db/db_rec.c` from **18%→~27% line / ~13%→~18% branch** in the subset
+  (merge/pgno/pg_trunc/relink no longer cold).
+
 ## The statistics (`*_stat_print`) surface
 
 The verbose `stat_print` / `DB_STAT_ALL` formatters (`env_stat.c`,

test/coverage/run_coverage.sh

@@ -73,6 +73,7 @@ bld="$root/build_unix"
 # a SIGSEGV off-by-one in __memp_remove_region -- see
 # test/coverage/MVCC-RESIZE-COVERAGE.md.)
 : "${COV_TESTS:=lock001: txn001: ssi001: ssi002: recd001:btree \
+  recd002:btree recd016:btree env007: \
   btree/test001 btree/test111 \
   hash/test001 hash/test006 hash/test010 hash/test025 hash/test077 \
   queue/test001 queue/test007 queue/test025 \
@@ -264,6 +265,39 @@ if [ "${COV_XA_UPG:-1}" = 1 ]; then
   echo "  .gcda files after xa/upg: $(find . -name '*.gcda' | wc -l)"
 fi
 
+# --- optional backup-API + compaction-recovery drivers (COV_BACKUP=1) --------
+# Two more standalone C drivers (same self-clean + hard-timeout shape as the
+# XA/upgrade drivers above) that light up code the Tcl suite cannot reach:
+#   env/env_backup.c   -- the hot-backup config + callback API
+#     (set/get_backup_config for all four enums, set/get_backup_callbacks).
+#     backup.tcl drives db_hotbackup, which calls DB_ENV->backup() with a NULL
+#     backup_handle, so env_backup.c stays 0%.  test/backup/run_backup_direct.sh
+#     compiles backup_direct.c, which calls those public entry points directly
+#     and then runs DB_ENV->backup()/dbbackup() with write callbacks installed
+#     -- lifting env_backup.c 0%->~97% and the backup->open/write/close callback
+#     branches of db/db_backup.c.
+#   db/db_rec.c        -- the btree-compaction + page-truncation recovery
+#     handlers __db_merge_recover / __db_pgno_recover / __db_pg_trunc_recover.
+#     No recd0NN test runs compaction under recovery, so those ~330 lines are
+#     cold.  test/db/run_recd_compact.sh compiles recd_compact.c, which fills+
+#     sparsifies a btree, compacts with DB_FREE_SPACE (logging merge/pgno/
+#     pg_trunc records) and re-opens under DB_RECOVER_FATAL to replay them.
+# Set COV_BACKUP=0 to skip.
+if [ "${COV_BACKUP:-1}" = 1 ]; then
+  echo "== run backup + compaction-recovery drivers (COV_BACKUP=1) =="
+  if sh "$root/test/backup/run_backup_direct.sh" >/tmp/cov-backup.log 2>&1; then
+    echo "PASS backup_direct"
+  else
+    echo "FAIL backup_direct (rc=$?)"; tail -5 /tmp/cov-backup.log
+  fi
+  if sh "$root/test/db/run_recd_compact.sh" >/tmp/cov-recdcompact.log 2>&1; then
+    echo "PASS recd_compact"
+  else
+    echo "FAIL recd_compact (rc=$?)"; tail -5 /tmp/cov-recdcompact.log
+  fi
+  echo "  .gcda files after backup/compact: $(find . -name '*.gcda' | wc -l)"
+fi
+
 # --- optional deadlock-detector + DB_REGISTER drivers (COV_DEAD_REG=1) -------
 # The deadlock DETECTOR (lock/lock_deadlock.c, __lock_detect) and the
 # process-registration + failchk-on-crash path (env/env_register.c,