examples: validate E2E manifest mappings

scripts/examples-e2e.rb

@@ -174,14 +174,31 @@ def summary
     private
 
     def document
-      @document ||= YAML.safe_load_file(manifest_path, aliases: false)
+      @document ||= YAML.safe_load_file(manifest_path, aliases: false).tap do |value|
+        unless value.is_a?(Hash)
+          raise ConfigurationError, "#{manifest_path}: manifest root must be a mapping"
+        end
+      end
+
     rescue Errno::ENOENT
       raise ConfigurationError, "example E2E manifest not found: #{manifest_path}"
     end
 
     def configured_examples
       value = document["examples"]
-      raise ConfigurationError, "manifest examples must be a mapping" unless value.is_a?(Hash)
+      unless value.is_a?(Hash)
+        raise ConfigurationError, "#{manifest_path}: manifest examples must be a mapping"
+      end
+
+      value.each do |path, configuration|
+        next if configuration.is_a?(Hash)
+
+        raise(
+          ConfigurationError,
+          "#{manifest_path}: #{path}: example configuration must be a mapping"
+        )
+      end
+
       value
     end
 

test/scripts/examples_e2e_test.rb

@@ -11,6 +11,34 @@
 require_relative "../../scripts/examples-e2e"
 
 class ExamplesE2EInventoryTest < Minitest::Test
+  def test_rejects_non_mapping_manifest_roots
+    [nil, [], "invalid"].each do |document|
+      with_manifest(document) do |inventory, manifest_path|
+        error = assert_raises(OpenAIExamplesE2E::ConfigurationError) { inventory.validate! }
+
+        assert_equal("#{manifest_path}: manifest root must be a mapping", error.message)
+      end
+    end
+  end
+
+  def test_rejects_non_mapping_example_configurations
+    [nil, [], "invalid"].each do |configuration|
+      document = {
+        "version" => 1,
+        "examples" => {"examples/example.rb" => configuration}
+      }
+
+      with_manifest(document) do |inventory, manifest_path|
+        error = assert_raises(OpenAIExamplesE2E::ConfigurationError) { inventory.validate! }
+
+        assert_equal(
+          "#{manifest_path}: examples/example.rb: example configuration must be a mapping",
+          error.message
+        )
+      end
+    end
+  end
+
   def test_rejects_non_string_exclusion_reasons
     [nil, false, 1, []].each do |reason|
       with_inventory(status: "excluded", reason: reason) do |inventory|
@@ -30,23 +58,28 @@ def test_accepts_a_non_empty_string_exclusion_reason
   private
 
   def with_inventory(status:, reason:)
+    document = {
+      "version" => 1,
+      "examples" => {
+        "examples/example.rb" => {"status" => status, "reason" => reason}
+      }
+    }
+
+    with_manifest(document) { |inventory, _manifest_path| yield(inventory) }
+  end
+
+  def with_manifest(document)
     Dir.mktmpdir("openai-examples-e2e-test") do |directory|
       root = Pathname(directory)
       example_path = root.join("examples/example.rb")
       example_path.dirname.mkpath
       example_path.write("# frozen_string_literal: true\n")
 
       manifest_path = root.join("examples/e2e.yml")
-      manifest_path.write(
-        YAML.dump(
-          "version" => 1,
-          "examples" => {
-            "examples/example.rb" => {"status" => status, "reason" => reason}
-          }
-        )
-      )
+      manifest_path.write(YAML.dump(document))
 
-      yield(OpenAIExamplesE2E::Inventory.new(root: root, manifest_path: manifest_path))
+      inventory = OpenAIExamplesE2E::Inventory.new(root: root, manifest_path: manifest_path)
+      yield(inventory, manifest_path)
     end
   end
 end