initial test-runner, with tests

Author: glennj

Dockerfile

@@ -1,8 +1,40 @@
-FROM alpine:3.18
+FROM ubuntu:24.04
 
-# install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+ENV LUA_VER="5.4.8"
+ENV LUA_CHECKSUM="4f18ddae154e793e46eeab727c59ef1c0c0c2b744e7b94219710d76f530629ae"
+ENV LUAROCKS_VER="3.12.0"
+ENV LUAROCKS_GPG_KEY="3FD8F43C2BB3C478"
 
+RUN apt-get update && \
+    apt-get install -y curl gcc jq make unzip gnupg git && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get purge --auto-remove && \
+    apt-get clean
+
+RUN curl -R -O -L http://www.lua.org/ftp/lua-${LUA_VER}.tar.gz && \
+    [ "$(sha256sum lua-${LUA_VER}.tar.gz | cut -d' ' -f1)" = "${LUA_CHECKSUM}" ] && \
+    tar -zxf lua-${LUA_VER}.tar.gz && \
+    cd lua-${LUA_VER} && \
+    make all install && \
+    cd .. && \
+    rm lua-${LUA_VER}.tar.gz && \
+    rm -rf lua-${LUA_VER}
+
+RUN curl -R -O -L https://luarocks.org/releases/luarocks-${LUAROCKS_VER}.tar.gz && \
+    curl -R -O -L https://luarocks.org/releases/luarocks-${LUAROCKS_VER}.tar.gz.asc && \
+    gpg --keyserver keyserver.ubuntu.com --recv-keys ${LUAROCKS_GPG_KEY} && \
+    gpg --verify luarocks-${LUAROCKS_VER}.tar.gz.asc luarocks-${LUAROCKS_VER}.tar.gz && \
+    tar -zxpf luarocks-${LUAROCKS_VER}.tar.gz && \
+    cd luarocks-${LUAROCKS_VER} && \
+    ./configure && make && make install && \
+    cd .. && \
+    rm luarocks-${LUAROCKS_VER}.tar.gz.asc && \
+    rm luarocks-${LUAROCKS_VER}.tar.gz && \
+    rm -rf luarocks-${LUAROCKS_VER}
+
+RUN luarocks install busted
+RUN luarocks install moonscript
+
+COPY . /opt/test-runner
 WORKDIR /opt/test-runner
-COPY . .
-ENTRYPOINT ["/opt/test-runner/bin/run.sh"]
+ENTRYPOINT ["/opt/test-runner/bin/run.moon"]

bin/run-in-docker.sh

@@ -1,27 +1,25 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
+set -e
 
 # Synopsis:
 # Run the test runner on a solution using the test runner Docker image.
 # The test runner Docker image is built automatically.
 
 # Arguments:
 # $1: exercise slug
-# $2: path to solution folder
-# $3: path to output directory
+# $2: absolute path to solution folder
+# $3: absolute path to output directory
 
 # Output:
 # Writes the test results to a results.json file in the passed-in output directory.
 # The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
 
 # Example:
-# ./bin/run-in-docker.sh two-fer path/to/solution/folder/ path/to/output/directory/
-
-# Stop executing when a command returns a non-zero return code
-set -e
+# ./bin/run-in-docker.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/
 
 # If any required arguments is missing, print the usage and exit
 if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
+    echo "usage: ./bin/run-in-docker.sh exercise-slug /absolute/path/to/solution/folder/ /absolute/path/to/output/directory/"
     exit 1
 fi
 
@@ -43,4 +41,4 @@ docker run \
     --mount type=bind,src="${solution_dir}",dst=/solution \
     --mount type=bind,src="${output_dir}",dst=/output \
     --mount type=tmpfs,dst=/tmp \
-    exercism/moonscript-test-runner "${slug}" /solution /output 
+    exercism/moonscript-test-runner "${slug}" /solution /output

bin/run-tests-in-docker.sh

@@ -1,4 +1,5 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
+set -e
 
 # Synopsis:
 # Test the test runner Docker image by running it against a predefined set of 
@@ -12,9 +13,6 @@
 # Example:
 # ./bin/run-tests-in-docker.sh
 
-# Stop executing when a command returns a non-zero return code
-set -e
-
 # Build the Docker image
 docker build --rm -t exercism/moonscript-test-runner .
 

bin/run-tests.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#! /bin/sh
 
 # Synopsis:
 # Test the test runner by running it against a predefined set of solutions 
@@ -17,21 +17,14 @@ exit_code=0
 for test_dir in tests/*; do
     test_dir_name=$(basename "${test_dir}")
     test_dir_path=$(realpath "${test_dir}")
-
-    bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
-
-    # OPTIONAL: Normalize the results file
-    # If the results.json file contains information that changes between 
-    # different test runs (e.g. timing information or paths), you should normalize
-    # the results file to allow the diff comparison below to work as expected
-
-    file="results.json"
-    expected_file="expected_${file}"
-    echo "${test_dir_name}: comparing ${file} to ${expected_file}"
-
-    if ! diff "${test_dir_path}/${file}" "${test_dir_path}/${expected_file}"; then
-        exit_code=1
-    fi
+    results_file="results.json"
+    results_file_path="${test_dir}/${results_file}"
+    expected_results_file="expected_results.json"
+    expected_results_file_path="${test_dir}/${expected_results_file}"
+
+    bin/run.moon "${test_dir_name}" "${test_dir}" "${test_dir}" \
+    && bin/test-result-compare.lua "${results_file_path}" "${expected_results_file_path}" \
+    || exit_code=1
 done
 
 exit ${exit_code}

bin/run.moon

@@ -0,0 +1,192 @@
+#! /usr/bin/env moon
+
+require 'moonscript'
+lfs = require 'lfs'
+json = (require 'dkjson').use_lpeg!
+getopt = require 'alt_getopt'
+local verbose
+
+import p from require 'moon'
+
+
+-- -----------------------------------------------------------
+show_help = (args) ->
+  print "Usage: #{args[0]} [-h] [-v]  slug  solution-dir  output-dir"
+  print "Where: -h   show this help"
+  print "       -v   verbose: show the output JSON"
+  os.exit!
+
+
+-- -----------------------------------------------------------
+file_exists = (path) ->
+  attrs = lfs.attributes path
+  not not attrs
+
+is_directory = (path) ->
+  attrs = lfs.attributes path
+  attrs and attrs.mode == 'directory'
+
+realpath = (path) ->
+  fh = io.popen "realpath #{path}"
+  dir = fh\read!
+  fh\close!
+  dir
+
+validate = (args) ->
+  show_help args unless #args == 3
+  {slug, src_dir, dest_dir} = args
+  assert slug != '', 'First arg, the slug, cannot be empty'
+  assert is_directory(src_dir), 'Second arg, the solution directory, must be a directory'
+  assert is_directory(dest_dir), 'Second arg, the output directory, must be a directory'
+
+  slug, realpath(src_dir), realpath(dest_dir)
+
+
+-- -----------------------------------------------------------
+run_tests = (slug, dir) ->
+  ok, err = lfs.chdir dir
+  assert ok, err
+
+  -- unskip tests
+  cmd = "perl -i.bak -pe 's{^\\s*\\Kpending\\b}{it}' *_spec.moon"
+  ok, result_type, status = os.execute cmd
+  assert ok
+
+  -- launch `busted`
+  fh = io.popen 'busted -o json', 'r'
+  json_output = fh\read 'a'
+  ok, exit_type, exit_status = fh\close!
+
+  if exit_type == 'signal'
+    return {
+      status: 'error',
+      message: json_output
+    }
+
+  data = json.decode json_output
+
+  if not data
+    return {
+      status: 'error',
+      message: json_output
+    }
+
+  if exit_status != 0 and #data.successes == 0 and #data.failures == 0 and #data.errors > 0
+    return {
+      status: 'error',
+      message: data.errors[1].message
+    }
+
+  results = {}
+
+  for test in *data.successes
+    results[test.element.name] = {
+      status: 'pass',
+      name: test.element.name,
+    }
+
+  for test in *data.failures
+    results[test.element.name] = {
+      status: 'fail',
+      name: test.element.name,
+      message: test.trace.message,
+    }
+
+  results
+
+
+-- -----------------------------------------------------------
+get_test_bodies = (slug, dir) ->
+  ok, err = lfs.chdir dir
+  assert ok, err
+
+  order = {}
+  bodies = {}
+
+  test_file = "#{slug\gsub('-', '_')}_spec.moon"
+  return unless file_exists test_file -- let `busted` handle the error messaging
+
+  fh = io.open test_file, 'r'
+
+  pattern = (word) -> '^%s+' .. word .. '%s+[\'"](.+)[\'"],%s+->'
+  patterns = it: pattern('it'), pending: pattern('pending')
+
+  local test_name
+  test_body = {}
+  in_test = false
+
+  for line in fh\lines!
+    if line\match '^%s+describe '
+      if test_name
+        bodies[test_name] = table.concat test_body, '\n'
+        test_body = {}
+        test_name = nil
+      in_test = false
+
+    m = line\match(patterns.it) or line\match(patterns.pending)
+    if not m
+      if in_test
+        table.insert test_body, line
+    else
+      table.insert order, m
+      if in_test
+        bodies[test_name] = table.concat test_body, '\n'
+        test_body = {}
+      test_name = m
+      in_test = true
+
+  fh\close!
+  bodies[test_name] = table.concat test_body, '\n'
+  order, bodies
+
+
+-- -----------------------------------------------------------
+write_results = (slug, test_results, names, bodies, dir) ->
+  ok, err = lfs.chdir dir
+  assert ok, "#{err}: #{dir}"
+  
+  results = version: 2, status: nil, tests: {}
+
+  if test_results.status
+    -- this was an error result
+    results.status = test_results.status
+    results.message = test_results.message
+
+  else
+    status = 'pass'
+    for name in *names
+      test = test_results[name]
+      assert test, "no test result for #{name}"
+      status = 'fail' if test.status == 'fail'
+      test.test_code = bodies[name]
+      table.insert results.tests, test
+    results.status = status
+
+  fh = io.open 'results.json', 'w'
+  fh\write (json.encode results) .. '\n'
+  fh\close!
+
+  os.execute "jq . results.json" if verbose
+
+
+-- -----------------------------------------------------------
+main = (args) ->
+  opts,optind = getopt.get_opts args, 'hv', {}
+
+  show_help args if opts.h
+  verbose = not not opts.v
+  table.remove args, 1 for _ = 1, optind - 1
+
+  slug, src_dir, dest_dir = validate args
+
+  print "#{slug}: testing ..."
+
+  test_names_ordered, test_code = get_test_bodies slug, src_dir
+
+  test_results = run_tests slug, src_dir
+
+  write_results slug, test_results, test_names_ordered, test_code, dest_dir
+
+  print "#{slug}: ... done"
+
+main arg

bin/test-result-compare.lua

@@ -0,0 +1,25 @@
+#!/usr/bin/env lua
+
+local json = require('dkjson')
+local tablex = require('pl.tablex')
+local pretty = require('pl.pretty')
+
+local function load_json(file)
+    local fd <close> = io.open(file)
+    return json.decode(fd:read('a'))
+end
+
+local result = load_json(arg[1])
+local expected = load_json(arg[2])
+
+if not tablex.deepcompare(result, expected) then
+    print('\n========[  RESULT  ]========\n')
+    print(pretty.write(result))
+
+    print('\n========[ EXPECTED ]========\n')
+    print(pretty.write(expected))
+
+    os.exit(1)
+end
+
+os.exit(0)

tests/.gitignore

@@ -0,0 +1,2 @@
+*.bak
+results.json

tests/all-fail/expected_results.json

@@ -0,0 +1,5 @@
+return {
+  default = {
+    ROOT = { '.' }
+  }
+}