From f5bbbf2d3aced551bf4c1640c95fb95122bb7348 Mon Sep 17 00:00:00 2001 From: Frankie Robertson Date: Sat, 1 Aug 2026 12:16:08 +0300 Subject: [PATCH 1/2] Add guard for presence of data in prepare_dataframe --- src/Sim/recorder.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sim/recorder.jl b/src/Sim/recorder.jl index 77fbbb2..4e1bc79 100644 --- a/src/Sim/recorder.jl +++ b/src/Sim/recorder.jl @@ -114,7 +114,7 @@ function prepare_dataframe(recording::CatRecording) Response = responses, ) for (name, value) in pairs(recording.data) - if value.data isa AbstractVector + if haskey(value, :data) && value.data isa AbstractVector label = haskey(value, :label) ? Symbol(value.label) : name cols = (; cols..., From ab71989b1ed302ceb6f95db6640bd97cd670e19c Mon Sep 17 00:00:00 2001 From: Frankie Robertson Date: Sat, 1 Aug 2026 13:17:30 +0300 Subject: [PATCH 2/2] Add some more termination conditions --- src/NextItemRules/NextItemRules.jl | 1 + src/NextItemRules/prelude/criteria.jl | 8 ++ src/TerminationConditions.jl | 45 +++++++++ test/runtests.jl | 1 + test/termination_conditions.jl | 137 ++++++++++++++++++++++++++ 5 files changed, 192 insertions(+) create mode 100644 test/termination_conditions.jl diff --git a/src/NextItemRules/NextItemRules.jl b/src/NextItemRules/NextItemRules.jl index edec020..4214fd1 100644 --- a/src/NextItemRules/NextItemRules.jl +++ b/src/NextItemRules/NextItemRules.jl @@ -64,6 +64,7 @@ export ObservedInformationPointwiseItemCriterion export RawEmpiricalInformationPointwiseItemCriterion export EmpiricalInformationPointwiseItemCriterion +public should_minimize public PointwiseNextItemRule, PointwiseFirstNextItemRule public WeightedStateMultiCriterion, WeightedItemMultiCriterion public GreedyForcedContentBalancer diff --git a/src/NextItemRules/prelude/criteria.jl b/src/NextItemRules/prelude/criteria.jl index f64c8ea..37ecc9f 100644 --- a/src/NextItemRules/prelude/criteria.jl +++ b/src/NextItemRules/prelude/criteria.jl @@ -32,6 +32,14 @@ function PointwiseItemCategoryCriterion(bits...) @returnsome find1_type(PointwiseItemCategoryCriterion, bits) typ->typ() end +""" + should_minimize(criterion) + +Whether lower values of a criterion are better. By convention criteria in this +package are minimised, so this defaults to `true`. +""" +should_minimize(::Union{ItemCriterion, CriterionBase}) = true + function init_thread(::ItemCriterion, ::TrackedResponses) nothing end diff --git a/src/TerminationConditions.jl b/src/TerminationConditions.jl index 080838f..10583b3 100644 --- a/src/TerminationConditions.jl +++ b/src/TerminationConditions.jl @@ -3,6 +3,7 @@ module TerminationConditions using DocStringExtensions: TYPEDEF, TYPEDFIELDS using FittedItemBanks: AbstractItemBank using ..Aggregators: TrackedResponses +using ..NextItemRules: StateCriterion, compute_criterion, should_minimize using ..ConfigBase import PsychometricsBazaarBase: power_summary using PsychometricsBazaarBase.ConfigTools: @returnsome, find1_instance @@ -11,6 +12,7 @@ import Base: show export TerminationCondition, FixedLength, TerminationTest export RunForever +export LengthBoundedTermination, StateCriterionThresholdTermination """ $(TYPEDEF) @@ -50,4 +52,47 @@ function (condition::RunForever)(::TrackedResponses, ::AbstractItemBank) return false end +""" +$(TYPEDEF) +$(TYPEDFIELDS) + +Wraps another termination condition so that the test always administers at least +`min_length` items and never more than `max_length` items. +""" +struct LengthBoundedTermination{InnerT <: TerminationCondition} <: TerminationCondition + min_length::Int64 + max_length::Int64 + termination_condition::InnerT +end +function (condition::LengthBoundedTermination)(responses::TrackedResponses, + items::AbstractItemBank) + nresp = length(responses) + return ( + (nresp >= condition.max_length) || + (nresp >= condition.min_length && condition.termination_condition(responses, items)) + ) +end + +""" +$(TYPEDEF) +$(TYPEDFIELDS) + +Terminates the test once a `StateCriterion` reaches `threshold`. When the +criterion is one which should be minimised, the test terminates once it drops to +or below the threshold, otherwise once it reaches or exceeds it. +""" +struct StateCriterionThresholdTermination{InnerT <: StateCriterion} <: TerminationCondition + threshold::Float64 + criterion::InnerT +end +function (condition::StateCriterionThresholdTermination)(responses::TrackedResponses, + ::AbstractItemBank) + value = compute_criterion(condition.criterion, responses) + if should_minimize(condition.criterion) + return value <= condition.threshold + else + return value >= condition.threshold + end +end + end diff --git a/test/runtests.jl b/test/runtests.jl index beb5390..ef85506 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -27,6 +27,7 @@ using .Dummy include("./jet.jl") include("./ability_estimator_1d.jl") include("./ability_estimator_2d.jl") + include("./termination_conditions.jl") include("./smoke.jl") include("./dt.jl") include("./stateful.jl") diff --git a/test/termination_conditions.jl b/test/termination_conditions.jl new file mode 100644 index 0000000..5d39d21 --- /dev/null +++ b/test/termination_conditions.jl @@ -0,0 +1,137 @@ +using ComputerAdaptiveTesting +using ComputerAdaptiveTesting.Aggregators +using ComputerAdaptiveTesting.Responses +using ComputerAdaptiveTesting.NextItemRules +import ComputerAdaptiveTesting.NextItemRules: should_minimize +using ComputerAdaptiveTesting.TerminationConditions +using FittedItemBanks +using PsychometricsBazaarBase.Integrators +using PsychometricsBazaarBase: power_summary +using Distributions + +""" +An item bank of 8 identical, well behaved items, so that the only thing which +varies between the states below is how many items have been administered. +""" +const term_item_bank = ItemBank2PL(zeros(8), fill(1.5, 8)) + +""" +Tracked responses with the first `num_responses` items administered. +""" +function mk_tracked_responses(num_responses) + responses = BareResponses( + ResponseType(term_item_bank), + collect(1:num_responses), + [isodd(idx) for idx in 1:num_responses] + ) + TrackedResponses(responses, term_item_bank, NullAbilityTracker()) +end + +const term_states = [mk_tracked_responses(num_responses) for num_responses in 0:8] + +""" +A `StateCriterion` counting administered items, used to check delegation +without dragging in integration. Larger is better, i.e. it is maximised. +""" +struct CountStateCriterion <: NextItemRules.StateCriterion end + +function NextItemRules.compute_criterion( + ::CountStateCriterion, tracked_responses::TrackedResponses) + Float64(length(tracked_responses)) +end + +should_minimize(::CountStateCriterion) = false + +@testset "termination_conditions" begin + @testset "FixedLength" begin + condition = FixedLength(3) + @test [condition(state, term_item_bank) for state in term_states] == + [false, false, false, true, true, true, true, true, true] + end + + @testset "FixedLength power_summary" begin + buf = IOBuffer() + power_summary(buf, FixedLength(3)) + @test occursin("3 items", String(take!(buf))) + end + + @testset "RunForever" begin + @test !any(RunForever()(state, term_item_bank) for state in term_states) + end + + @testset "TerminationTest delegates" begin + seen = [] + condition = TerminationTest(function (responses, items) + push!(seen, (length(responses), items)) + length(responses) == 2 + end) + @test !condition(term_states[1], term_item_bank) + @test condition(term_states[3], term_item_bank) + @test seen == [(0, term_item_bank), (2, term_item_bank)] + end + + @testset "TerminationCondition implicit constructor" begin + condition = FixedLength(3) + @test TerminationCondition(condition) === condition + @test TerminationCondition("unrelated") === nothing + end + + @testset "LengthBoundedTermination stops at max_length" begin + condition = LengthBoundedTermination(2, 5, RunForever()) + @test [condition(state, term_item_bank) for state in term_states] == + [false, false, false, false, false, true, true, true, true] + end + + @testset "LengthBoundedTermination respects min_length" begin + condition = LengthBoundedTermination( + 3, 6, TerminationTest((responses, items) -> true)) + @test [condition(state, term_item_bank) for state in term_states] == + [false, false, false, true, true, true, true, true, true] + end + + @testset "LengthBoundedTermination passes through inner condition" begin + condition = LengthBoundedTermination(2, 7, FixedLength(4)) + @test [condition(state, term_item_bank) for state in term_states] == + [false, false, false, false, true, true, true, true, true] + end + + @testset "LengthBoundedTermination max_length wins over min_length" begin + # A degenerate configuration: the maximum is reached before the minimum + condition = LengthBoundedTermination(5, 2, RunForever()) + @test condition(term_states[3], term_item_bank) + end + + @testset "StateCriterionThresholdTermination minimised criterion" begin + criterion = AbilityVariance( + LikelihoodAbilityEstimator(), + AbilityIntegrator(FixedGKIntegrator(-6.0, 6.0, 61)) + ) + @test should_minimize(criterion) + variances = [compute_criterion(criterion, state) for state in term_states[2:end]] + # Sanity check: the variance shrinks as responses come in + @test variances[end] < variances[1] + + threshold = (variances[4] + variances[5]) / 2 + condition = StateCriterionThresholdTermination(threshold, criterion) + @test !condition(term_states[5], term_item_bank) + @test condition(term_states[6], term_item_bank) + + # Threshold above every observed value terminates immediately + @test StateCriterionThresholdTermination(maximum(variances) + 1.0, criterion)( + term_states[2], term_item_bank) + # Threshold below every observed value never terminates + @test !StateCriterionThresholdTermination(0.0, criterion)( + term_states[end], term_item_bank) + end + + @testset "StateCriterionThresholdTermination maximised criterion" begin + condition = StateCriterionThresholdTermination(3.0, CountStateCriterion()) + @test [condition(state, term_item_bank) for state in term_states] == + [false, false, false, true, true, true, true, true, true] + end + + @testset "StateCriterionThresholdTermination exactly at threshold" begin + @test StateCriterionThresholdTermination(2.0, CountStateCriterion())( + term_states[3], term_item_bank) + end +end