add new algorithm for systematic sampling

lib/split.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'redis'
 
+require 'split/algorithms/systematic_sampling'
 require 'split/algorithms/block_randomization'
 require 'split/algorithms/weighted_sample'
 require 'split/algorithms/whiplash'

lib/split/algorithms/systematic_sampling.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+module Split
+  module Algorithms
+    module SystematicSampling
+      def self.choose_alternative(experiment)
+        count = experiment.next_cohorting_block_index
+
+        block_length = experiment.cohorting_block_magnitude * experiment.alternatives.length
+        block_num, index = count.divmod block_length
+
+        r = Random.new(block_num + experiment.cohorting_block_seed)
+        block = (experiment.alternatives*experiment.cohorting_block_magnitude).shuffle(random: r)
+        block[index]
+      end
+    end
+  end
+end

lib/split/configuration.rb

@@ -164,7 +164,9 @@ def normalized_experiments
           algorithm: value_for(settings, :algorithm),
           resettable: value_for(settings, :resettable),
           friendly_name: value_for(settings, :friendly_name),
-          retain_user_alternatives_after_reset:  value_for(settings, :retain_user_alternatives_after_reset)
+          retain_user_alternatives_after_reset:  value_for(settings, :retain_user_alternatives_after_reset),
+          cohorting_block_seed: value_for(settings, :cohorting_block_seed),
+          cohorting_block_magnitude: value_for(settings, :cohorting_block_magnitude)
         }
 
         experiment_data.each do |name, value|

lib/split/experiment.rb

@@ -9,6 +9,8 @@ class Experiment
     attr_accessor :alternative_probabilities
     attr_accessor :metadata
     attr_accessor :friendly_name
+    attr_accessor :cohorting_block_seed
+    attr_accessor :cohorting_block_magnitude
 
     attr_reader :alternatives
     attr_reader :resettable
@@ -17,6 +19,7 @@ class Experiment
     DEFAULT_OPTIONS = {
       :resettable => true,
       :retain_user_alternatives_after_reset => false,
+      :cohorting_block_magnitude => 1
     }
 
     def initialize(name, options = {})
@@ -43,6 +46,11 @@ def set_alternatives_and_options(options)
       self.metadata = options_with_defaults[:metadata]
       self.friendly_name = options_with_defaults[:friendly_name] || @name
       self.retain_user_alternatives_after_reset = options_with_defaults[:retain_user_alternatives_after_reset]
+
+      if self.algorithm == Split::Algorithms::SystematicSampling
+        self.cohorting_block_seed = options_with_defaults[:cohorting_block_seed] || self.name.to_i(36)
+        self.cohorting_block_magnitude = options_with_defaults[:cohorting_block_magnitude]
+      end
     end
 
     def extract_alternatives_from_options(options)
@@ -64,6 +72,8 @@ def extract_alternatives_from_options(options)
           options[:algorithm] = exp_config[:algorithm]
           options[:friendly_name] = exp_config[:friendly_name]
           options[:retain_user_alternatives_after_reset] = exp_config[:retain_user_alternatives_after_reset]
+          options[:cohorting_block_seed] = exp_config[:cohorting_block_seed]
+          options[:cohorting_block_magnitude] = exp_config[:cohorting_block_magnitude]
         end
       end
 
@@ -232,6 +242,14 @@ def friendly_name_key
       "#{name}:friendly_name"
     end
 
+    def cohorting_block_seed_key
+      "#{name}:cohorting_block_seed"
+    end
+
+    def cohorting_block_magnitude_key
+      "#{name}:cohorting_block_magnitude"
+    end
+
     def resettable?
       resettable
     end
@@ -266,6 +284,8 @@ def load_from_redis
 
       options = {
         retain_user_alternatives_after_reset: exp_config['retain_user_alternatives_after_reset'],
+        cohorting_block_seed: load_cohorting_block_seed_from_redis,
+        cohorting_block_magnitude: load_cohorting_block_magnitude_from_redis,
         resettable: exp_config['resettable'],
         algorithm: exp_config['algorithm'],
         friendly_name: load_friendly_name_from_redis,
@@ -423,6 +443,10 @@ def enable_cohorting
       redis.hset(experiment_config_key, :cohorting, false)
     end
 
+    def next_cohorting_block_index
+      Split.redis.incr("#{name}:cohorting_block_index") - 1
+    end
+
     protected
 
     def experiment_config_key
@@ -446,6 +470,14 @@ def load_friendly_name_from_redis
       redis.get(friendly_name_key)
     end
 
+    def load_cohorting_block_seed_from_redis
+      redis.get(cohorting_block_seed_key).to_i
+    end
+
+    def load_cohorting_block_magnitude_from_redis
+      redis.get(cohorting_block_magnitude_key).to_i
+    end
+
     def load_alternatives_from_configuration
       alts = Split.configuration.experiment_for(@name)[:alternatives]
       raise ArgumentError, "Experiment configuration is missing :alternatives array" unless alts
@@ -492,6 +524,8 @@ def persist_experiment_configuration
       goals_collection.save
       redis.set(metadata_key, @metadata.to_json) unless @metadata.nil?
       redis.set(friendly_name_key, self.friendly_name)
+      redis.set(cohorting_block_seed_key, self.cohorting_block_seed)
+      redis.set(cohorting_block_magnitude_key, self.cohorting_block_magnitude)
     end
 
     def remove_experiment_configuration

spec/algorithms/systematic_sampling_spec.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+require "spec_helper"
+
+describe Split::Algorithms::SystematicSampling do
+  let(:experiment) do
+    Split::Experiment.new(
+      'link_color',
+      :alternatives => ['red', 'blue', 'green'],
+      :algorithm => Split::Algorithms::SystematicSampling,
+      :cohorting_block_magnitude => 2
+    )
+  end
+
+  it "should return an alternative" do
+    expect(Split::Algorithms::SystematicSampling.choose_alternative(experiment).class).to eq(Split::Alternative)
+  end
+
+  context "experiments with a random seed" do
+    it "cohorts the first block of users equally into each alternative" do
+      results = {'red' => 0, 'blue' => 0, 'green' => 0}
+      6.times do
+        results[Split::Algorithms::SystematicSampling.choose_alternative(experiment).name] += 1
+      end
+
+      expect(experiment.cohorting_block_magnitude * experiment.alternatives.length).to eq(6)
+      expect(results).to eq({'red' => 2, 'blue' => 2, 'green' => 2})
+    end
+
+    it "cohorts the second block of users equally into each alternative" do
+      6.times do
+        Split::Algorithms::SystematicSampling.choose_alternative(experiment).name
+      end
+
+      results = {'red' => 0, 'blue' => 0, 'green' => 0}
+      6.times do
+        results[Split::Algorithms::SystematicSampling.choose_alternative(experiment).name] += 1
+      end
+
+      expect(experiment.cohorting_block_magnitude * experiment.alternatives.length).to eq(6)
+      expect(results).to eq({'red' => 2, 'blue' => 2, 'green' => 2})
+    end
+  end
+
+  context "experiments with set seed" do
+    let(:seeded_experiment1) do
+      Split::Experiment.new(
+        'link_color',
+        :alternatives => ['red', 'blue', 'green'],
+        :algorithm => Split::Algorithms::SystematicSampling,
+        :cohorting_block_seed => 1234
+      )
+    end
+
+    let(:seeded_experiment2) do
+      Split::Experiment.new('link_highlight',
+        :alternatives => ['red', 'blue', 'green'],
+        :algorithm => Split::Algorithms::SystematicSampling,
+        :cohorting_block_seed => 1234)
+    end
+
+    it "cohorts users in a set order" do
+      results1 = []
+      results2 = []
+
+      12.times do
+        results1 << Split::Algorithms::SystematicSampling.choose_alternative(seeded_experiment1).name
+      end
+
+      12.times do
+        results2 << Split::Algorithms::SystematicSampling.choose_alternative(seeded_experiment2).name
+      end
+
+      expect(seeded_experiment1.cohorting_block_seed).to eq(seeded_experiment2.cohorting_block_seed)
+      expect(results1).to eq(results2)
+    end
+  end
+end

spec/configuration_spec.rb

@@ -135,6 +135,9 @@
                   percent: 23
               resettable: false
               retain_user_alternatives_after_reset: true
+              algorithm: Split::Algorithms::SystematicSampling
+              cohorting_block_seed: 999
+              cohorting_block_magnitude: 3
               metric: my_metric
             another_experiment:
               alternatives:
@@ -145,8 +148,9 @@
         end
 
         it "should normalize experiments" do
-          expect(@config.normalized_experiments).to eq({:my_experiment=>{:resettable=>false,:retain_user_alternatives_after_reset=>true,:alternatives=>[{"Control Opt"=>0.67},
-            [{"Alt One"=>0.1}, {"Alt Two"=>0.23}]]}, :another_experiment=>{:alternatives=>["a", ["b"]]}})
+          expect(@config.normalized_experiments).to eq({:my_experiment=>{:resettable=>false,:retain_user_alternatives_after_reset=>true, :cohorting_block_magnitude=>3,
+            :algorithm=>"Split::Algorithms::SystematicSampling", :cohorting_block_seed=>999,:alternatives=>[{"Control Opt"=>0.67},[{"Alt One"=>0.1}, {"Alt Two"=>0.23}]]},
+            :another_experiment=>{:alternatives=>["a", ["b"]]}})
         end
 
         it "should recognize metrics" do

spec/experiment_spec.rb

@@ -128,6 +128,16 @@ def alternative(color)
       expect(experiment.retain_user_alternatives_after_reset).to be_truthy
     end
 
+    it "should be possible to make a SystematicSampling algorithm experiment with a seeded cohorting block" do
+      experiment = Split::Experiment.new("basket_text", :alternatives => ["Basket", "Cart"], algorithm: Split::Algorithms::SystematicSampling, :cohorting_block_seed => 123)
+      expect(experiment.cohorting_block_seed).to eq(123)
+    end
+
+    it "should be possible to make a SystematicSampling algorithm experiment with a custom cohorting block magnitude" do
+      experiment = Split::Experiment.new("basket_text", :alternatives => ["Basket", "Cart"], algorithm: Split::Algorithms::SystematicSampling, :cohorting_block_magnitude => 4)
+      expect(experiment.cohorting_block_magnitude).to eq(4)
+    end
+
     it "sets friendly_name" do
       experiment = Split::Experiment.new('basket_text', :alternatives => ['Basket', "Cart"], :friendly_name => "foo")
       expect(experiment.friendly_name).to eq("foo")
@@ -151,6 +161,18 @@ def alternative(color)
         expect(Split::Experiment.new(experiment_name).retain_user_alternatives_after_reset).to eq(false)
       end
 
+      it 'when the experiment is using SystematicSampling algorithm, assigns cohorting_block_magnitude to a default value' do
+        expect(Split::Experiment.new('systematic_sampling_exp', algorithm: Split::Algorithms::SystematicSampling).cohorting_block_magnitude).to eq(1)
+      end
+
+      it 'when the experiment is using SystematicSampling algorithm, assigns cohorting_block_seed to a default value' do
+        expect(Split::Experiment.new('systematic_sampling_exp', algorithm: Split::Algorithms::SystematicSampling).cohorting_block_seed)
+          .to eq(387211932128858527590062598078109)
+
+        expect(Split::Experiment.new('systematic_sampling_exp2', algorithm: Split::Algorithms::SystematicSampling).cohorting_block_seed)
+          .to eq(13939629556638906993242253530811926)
+      end
+
       it "sets friendly_name" do
         expect(Split::Experiment.new(experiment_name).friendly_name).to eq("foo")
       end
@@ -185,6 +207,24 @@ def alternative(color)
       expect(e.retain_user_alternatives_after_reset).to be_truthy
     end
 
+    it "should persist cohorting_block_magnitude" do
+      experiment = Split::Experiment.new("basket_text", :alternatives => ['Basket', "Cart"], algorithm: Split::Algorithms::SystematicSampling, :cohorting_block_magnitude => 2)
+      experiment.save
+
+      e = Split::ExperimentCatalog.find("basket_text")
+      expect(e).to eq(experiment)
+      expect(e.cohorting_block_magnitude).to eq(2)
+    end
+
+    it "should persist cohorting_block_seed" do
+      experiment = Split::Experiment.new("basket_text", :alternatives => ['Basket', "Cart"], algorithm: Split::Algorithms::SystematicSampling, :cohorting_block_seed => 12345)
+      experiment.save
+
+      e = Split::ExperimentCatalog.find("basket_text")
+      expect(e).to eq(experiment)
+      expect(e.cohorting_block_seed).to eq(12345)
+    end
+
     describe '#metadata' do
       let(:experiment) { Split::Experiment.new('basket_text', :alternatives => ['Basket', "Cart"], :algorithm => Split::Algorithms::Whiplash, :metadata => meta) }
       context 'simple hash' do
@@ -427,6 +467,23 @@ def alternative(color)
     end
   end
 
+  describe '#next_cohorting_block_index' do
+    it 'increments each call and starts at 0' do
+      expect(experiment.next_cohorting_block_index).to eq(0)
+      expect(experiment.next_cohorting_block_index).to eq(1)
+      expect(experiment.next_cohorting_block_index).to eq(2)
+      expect(experiment.next_cohorting_block_index).to eq(3)
+    end
+
+    it 'persists value' do
+      expect(experiment.next_cohorting_block_index).to eq(0)
+      experiment.save
+
+      e = Split::ExperimentCatalog.find("link_color")
+      expect(e.next_cohorting_block_index).to eq(1)
+    end
+  end
+
   describe '#next_alternative' do
     context 'with multiple alternatives' do
       let(:experiment) { Split::ExperimentCatalog.find_or_create('link_color', 'blue', 'red', 'green') }