Add external measure manager for non-gem repos

.github/workflows/ci.yml

@@ -25,9 +25,5 @@ jobs:
         gem install bundler -v 2.4.10
         bundle config set --local path "$GITHUB_WORKSPACE/.bundle/install"
         bundle install
-    - name: Run Integration Tests
-      run: bundle exec rspec spec/tests/integration/write_and_run_osws_spec.rb
-      if: always()
-    - name: Run Unit Tests
-      run: bundle exec rspec spec/tests/unit/buildingsync_reader_spec.rb
-      if: always()
+    - name: Run Tests
+      run: bundle exec rspec

Gemfile.lock

@@ -158,6 +158,11 @@ GEM
       unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
+    rubocop-checkstyle_formatter (0.6.0)
+      rubocop (>= 1.14.0)
+    rubocop-performance (1.20.0)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
     ruby-ole (1.2.13.1)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
@@ -191,6 +196,8 @@ DEPENDENCIES
   rake (~> 13.0)
   rspec (~> 3.13)
   rubocop (= 1.50)
+  rubocop-checkstyle_formatter (= 0.6.0)
+  rubocop-performance (= 1.20.0)
 
 BUNDLED WITH
    2.4.10

README.md

@@ -21,6 +21,30 @@ BuildingSync OpenStudio Simulator (BOSS) takes in BuildingSync files, creates Op
     bundle install
     ```
 
+4. Install external non-gem measure repositories declared in the shipped manifest.
+
+    ```bash
+    bundle exec rake measures:install_external
+    ```
+
+5. (Optional) List resolved external measure roots.
+
+    ```bash
+    bundle exec rake measures:list_external
+    ```
+
+The shipped manifest lives at [config/external_measure_repos.yml](config/external_measure_repos.yml) and is preconfigured with:
+
+- local measure roots: `lib/measures` (first in precedence)
+- external repository: comstock (`measures` and `resources/measures`)
+
+Measure path precedence is:
+
+1. local measure roots from [config/external_measure_repos.yml](config/external_measure_repos.yml)
+2. gem-provided measure directories
+3. external non-gem repository measure roots from [config/external_measure_repos.yml](config/external_measure_repos.yml)
+
+The generated `in.osw` will include these paths in order under `measure_paths`.
 
 ## OpenStudio Compatibility Version
 
@@ -97,6 +121,11 @@ bundle exec boss write_baseline_osw BUILDINGSYNC_FILE -o OUTPUT_PATH [options]
 bundle exec boss run_osw BUILDINGSYNC_FILE -o OUTPUT_PATH [options]
 ```
 
+To get help text, type the following:
+```bash
+bundle exec boss help [COMMAND]
+```
+
 ### Options
 
 - `-o`, `--output_path` (required): directory where BOSS writes output (for example, use an XML-specific folder such as `output/v2.7.0/L100_Audit-1.0.0`)

Rakefile

@@ -6,10 +6,37 @@
 
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
+require_relative './lib/BOSS/constants'
+require_relative './lib/BOSS/external_measure_repo_manager'
 
 RSpec::Core::RakeTask.new(:spec)
 
 require 'rubocop/rake_task'
 RuboCop::RakeTask.new
 
-task default: :spec
+task default: :spec
+
+namespace :measures do
+  desc 'Clone/fetch external non-gem measure repositories declared in config/external_measure_repos.yml'
+  task :install_external do
+    manager = BOSS::ExternalMeasureRepoManager.new
+
+    if !manager.manifest_exists?
+      puts "No external measure manifest found at #{EXTERNAL_MEASURE_REPOS_MANIFEST_PATH}"
+      next
+    end
+
+    dirs = manager.install_all
+    puts "Installed external measure roots (#{dirs.length}):"
+    dirs.each { |dir| puts "  - #{dir}" }
+  end
+
+  desc 'List resolved external non-gem measure directories'
+  task :list_external do
+    manager = BOSS::ExternalMeasureRepoManager.new
+    dirs = manager.resolved_measure_directories
+
+    puts "External measure roots (#{dirs.length}):"
+    dirs.each { |dir| puts "  - #{dir}" }
+  end
+end

config/external_measure_repos.yml

@@ -0,0 +1,27 @@
+# External non-gem measure repositories.
+#
+# Precedence in workflow measure lookup is:
+# 1) local_measure_roots below
+# 2) gem-provided measure dirs
+# 3) external repos listed here
+#
+# local_measure_roots are resolved relative to this file.
+# The default shipped configuration keeps this repository's local measures first.
+local_measure_roots:
+  - ../lib/measures
+
+# For each repo:
+# - name: local checkout folder name under vendor/external_measures
+# - url: git repository URL
+# - ref: pinned tag/branch/commit to checkout
+# - measure_roots: one or more paths inside the repo that contain measure directories
+#
+
+# The below measures are needed for the BOSS L200 workflow.
+repos:
+  - name: comstock
+    url: https://github.com/NatLabRockies/comstock.git
+    ref: main
+    measure_roots:
+      - measures
+      - resources/measures

lib/BOSS/boss.rb

@@ -9,6 +9,7 @@
 require 'fileutils'
 require 'BOSS/buildingsync_reader/buildingsync_reader'
 require 'BOSS/osw_arg_populator'
+require 'BOSS/external_measure_repo_manager'
 
 require 'openstudio/common_measures'
 require 'openstudio/model_articulation'
@@ -56,6 +57,31 @@ def write_baseline_osw
       OSWArgPopulator::populate_set_electric_equipment_loads_by_epd_args(baseline_osw, @bsync_reader)
       OSWArgPopulator::populate_openstudio_results_args(baseline_osw, @bsync_reader)
 
+      # Gather extension-derived paths (typically gem-based measures/files) from ObjectSpace.
+      OpenStudio::Extension.configure_osw(baseline_osw)
+
+      # Force ordering: local BOSS measures, extension-discovered (gems), then external repos.
+      manager = ExternalMeasureRepoManager.new
+      local_measure_dirs = manager.local_measure_directories
+      external_measure_dirs = manager.resolved_measure_directories
+      discovered_measure_dirs = baseline_osw[:measure_paths] || []
+
+      discovered_nonlocal_nonexternal = discovered_measure_dirs.reject do |path|
+        local_measure_dirs.include?(path) || external_measure_dirs.include?(path)
+      end
+
+      baseline_osw[:measure_paths] = ordered_unique(local_measure_dirs + discovered_nonlocal_nonexternal + external_measure_dirs)
+
+      local_file_paths = local_measure_dirs.map { |path| measure_path_to_files_path(path) }
+      external_file_paths = external_measure_dirs.map { |path| measure_path_to_files_path(path) }
+      discovered_file_paths = baseline_osw[:file_paths] || []
+
+      discovered_file_nonlocal_nonexternal = discovered_file_paths.reject do |path|
+        local_file_paths.include?(path) || external_file_paths.include?(path)
+      end
+
+      baseline_osw[:file_paths] = ordered_unique(local_file_paths + discovered_file_nonlocal_nonexternal + external_file_paths)
+
       # write to file
       workflow_dir = File.join(@output_dir, 'baseline')
       FileUtils.mkdir_p(workflow_dir)
@@ -85,5 +111,23 @@ def run_baseline_osw
       # run the baseline osm
       return runner.run_osws([baseline_osw_path])
     end
+
+    private
+
+    def measure_path_to_files_path(measure_path)
+      path = measure_path.to_s
+      replaced = path.sub(%r{/measures/?$}, '/files')
+      return replaced if replaced != path
+
+      File.expand_path(File.join(path, '..', 'files'))
+    end
+
+    def ordered_unique(paths)
+      (paths || []).each_with_object([]) do |path, acc|
+        next if path.nil? || path.empty? || acc.include?(path)
+
+        acc << path
+      end
+    end
   end
 end

lib/BOSS/constants.rb

@@ -12,9 +12,10 @@
 SCHEMA_2_7_0_URL = 'https://raw.githubusercontent.com/BuildingSync/schema/v2.7.0/BuildingSync.xsd'
 EMPTY_BASELINE_OSW_PATH = File.expand_path(File.join(__dir__, 'empty_baseline.osw'))
 BUILDING_TYPES_BY_OCCUPANCY_CLASSIFICATION_PATH = File.expand_path(File.join(__dir__, 'buildingsync_reader/building_types_by_occupancy_classification.json'))
-
+LOCAL_MEASURES_DIR = File.expand_path(File.join(__dir__, '..', 'measures'))
 WEATHER_DIR = File.expand_path(File.join(__dir__, '../../weather'))
-
+EXTERNAL_MEASURE_REPOS_MANIFEST_PATH = File.expand_path(File.join(__dir__, '..', '..', 'config', 'external_measure_repos.yml'))
+EXTERNAL_MEASURE_REPOS_INSTALL_DIR = File.expand_path(File.join(__dir__, '..', '..', 'vendor', 'external_measures'))
 
 # Standards strings
 ASHRAE90_1 = 'ASHRAE90.1'

lib/BOSS/external_measure_repo_manager.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+# *******************************************************************************
+# OpenStudio(R), Copyright (c) Alliance for Energy Innovation, LLC.
+# See also https://github.com/BuildingSync/BuildingSync-gem/blob/develop/LICENSE.md
+# *******************************************************************************
+
+require 'fileutils'
+require 'open3'
+require 'yaml'
+
+module BOSS
+  # Manages non-gem measure repositories declared in a manifest file.
+  class ExternalMeasureRepoManager
+    def initialize(manifest_path: EXTERNAL_MEASURE_REPOS_MANIFEST_PATH, install_dir: EXTERNAL_MEASURE_REPOS_INSTALL_DIR)
+      @manifest_path = manifest_path
+      @install_dir = install_dir
+    end
+
+    def manifest_exists?
+      File.file?(@manifest_path)
+    end
+
+    def configured?
+      !repo_entries.empty?
+    end
+
+    def local_measure_directories
+      return [LOCAL_MEASURES_DIR] if !manifest_exists?
+
+      roots = manifest['local_measure_roots']
+      roots = [LOCAL_MEASURES_DIR] if !roots.is_a?(Array) || roots.empty?
+
+      directories = roots.map do |path|
+        File.expand_path(path, File.expand_path('..', @manifest_path))
+      end
+
+      directories.select { |path| Dir.exist?(path) }.uniq
+    end
+
+    def install_all
+      return [] if !manifest_exists?
+
+      FileUtils.mkdir_p(@install_dir)
+
+      repo_entries.each do |repo_entry|
+        install_repo(repo_entry)
+      end
+
+      resolved_measure_directories
+    end
+
+    def resolved_measure_directories
+      return [] if !manifest_exists?
+
+      directories = []
+      repo_entries.each do |repo_entry|
+        repo_root = repo_checkout_dir(repo_entry)
+        if !Dir.exist?(repo_root)
+          message = "Repository '#{repo_entry['name']}' is not installed at #{repo_root}. Run rake measures:install_external."
+          if defined?(OpenStudio) && OpenStudio.respond_to?(:logFree)
+            OpenStudio.logFree(OpenStudio::Warn, 'BOSS.ExternalMeasureRepoManager.resolved_measure_directories', message)
+          else
+            warn("BOSS.ExternalMeasureRepoManager.resolved_measure_directories: #{message}")
+          end
+          next
+        end
+
+        repo_entry['measure_roots'].each do |root_rel_path|
+          root_abs_path = File.expand_path(File.join(repo_root, root_rel_path))
+          if Dir.exist?(root_abs_path)
+            directories << root_abs_path
+          else
+            message = "Configured measure root '#{root_rel_path}' does not exist for repository '#{repo_entry['name']}' (#{root_abs_path})."
+            if defined?(OpenStudio) && OpenStudio.respond_to?(:logFree)
+              OpenStudio.logFree(OpenStudio::Warn, 'BOSS.ExternalMeasureRepoManager.resolved_measure_directories', message)
+            else
+              warn("BOSS.ExternalMeasureRepoManager.resolved_measure_directories: #{message}")
+            end
+          end
+        end
+      end
+
+      ordered_unique(directories)
+    end
+
+    private
+
+    def manifest
+      @manifest ||= begin
+        parsed = YAML.safe_load(File.read(@manifest_path), permitted_classes: [], aliases: false)
+        parsed.is_a?(Hash) ? parsed : {}
+      end
+    end
+
+    def repo_entries
+      return [] if !manifest_exists?
+
+      repos = manifest.is_a?(Hash) ? manifest['repos'] : nil
+      return [] if repos.nil?
+      raise StandardError, "Expected 'repos' array in #{@manifest_path}" if !repos.is_a?(Array)
+
+      repos.map do |repo|
+        validate_repo_entry(repo)
+      end
+    end
+
+    def validate_repo_entry(repo)
+      if !repo.is_a?(Hash)
+        raise StandardError, "Each repo entry in #{@manifest_path} must be a map"
+      end
+
+      required = %w[name url ref measure_roots]
+      missing = required.select { |key| repo[key].nil? || repo[key].to_s.empty? }
+      if !missing.empty?
+        raise StandardError, "Repo entry is missing required keys #{missing.join(', ')} in #{@manifest_path}"
+      end
+
+      # Prevent path traversal / unexpected nesting in checkout dir.
+      if !repo['name'].to_s.match?(/\A[\w.-]+\z/)
+        raise StandardError, "Repo name '#{repo['name']}' must match /\\A[\\w.-]+\\z/ in #{@manifest_path}"
+      end
+
+      if !repo['measure_roots'].is_a?(Array) || repo['measure_roots'].empty?
+        raise StandardError, "Repo '#{repo['name']}' must define a non-empty measure_roots array"
+      end
+      repo
+    end
+
+    def install_repo(repo_entry)
+      repo_dir = repo_checkout_dir(repo_entry)
+      measure_roots = repo_entry['measure_roots']
+
+      if Dir.exist?(File.join(repo_dir, '.git'))
+        run_git(%W[-C #{repo_dir} remote set-url origin #{repo_entry['url']}])
+      else
+        FileUtils.mkdir_p(File.dirname(repo_dir))
+        run_git(%W[clone --filter=blob:none --no-checkout #{repo_entry['url']} #{repo_dir}])
+      end
+
+      run_git(%W[-C #{repo_dir} sparse-checkout init --cone])
+      run_git(['-C', repo_dir, 'sparse-checkout', 'set', *measure_roots])
+      run_git(%W[-C #{repo_dir} fetch --depth 1 origin #{repo_entry['ref']}])
+      run_git(%W[-C #{repo_dir} checkout --force FETCH_HEAD])
+    end
+
+    def repo_checkout_dir(repo_entry)
+      File.join(@install_dir, repo_entry['name'])
+    end
+
+    def run_git(args)
+      stdout, stderr, status = Open3.capture3('git', *args)
+      if !status.success?
+        raise StandardError, "git #{args.join(' ')} failed:\n#{stdout}\n#{stderr}"
+      end
+    end
+
+    def ordered_unique(paths)
+      seen = {}
+      paths.each_with_object([]) do |path, acc|
+        next if seen[path]
+
+        seen[path] = true
+        acc << path
+      end
+    end
+  end
+end