Change-point tests

.coveragerc

@@ -2,6 +2,7 @@
 source = pints
 omit = 
     pints/tests/*
+    pints/functionaltests/*
 
 [report]
 exclude_lines =

.github/workflows/copyright-test.yml

@@ -12,6 +12,7 @@ on:
 jobs:
 
   build-and-test:
+    if: github.event.pull_request.draft == false
     name: Copyright
     runs-on: ubuntu-latest
 

.github/workflows/coverage-test.yml

@@ -12,6 +12,7 @@ on:
 jobs:
 
   build-and-test:
+    if: github.event.pull_request.draft == false
     name: Coverage
     runs-on: ubuntu-latest
 

.github/workflows/docs-test.yml

@@ -12,6 +12,7 @@ on:
 jobs:
 
   build-and-test:
+    if: github.event.pull_request.draft == false
     name: Docs
     runs-on: ubuntu-latest
 

.github/workflows/functional-tests.yml

@@ -0,0 +1,58 @@
+name: Functional testing
+
+on:
+  push:
+    branches:
+      - issue-1294-functional-testing-module
+
+jobs:
+
+  run-functional-testing:
+    name: Functional testing
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out PINTS repository
+        uses: actions/checkout@v2
+        with:
+          path: 'pints'
+
+      - name: Check out functional-testing-2 repository
+        uses: actions/checkout@v2
+        with:
+          repository: 'pints-team/functional-testing-2'
+          ref: 'new-infra'
+          path: 'functional-testing-2'
+          token: ${{ secrets.FUNCTIONAL_TESTING_TOKEN }}
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+          architecture: x64
+
+      - name: install pints
+        run: |
+          python --version
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install .
+        working-directory: 'pints'
+
+      - name: install functional-testing-2
+        run: |
+          python -m pip install .
+        working-directory: 'functional-testing-2'
+
+      - name: run functional testing
+        run: |
+          python run_all_the_things.py
+        working-directory: 'functional-testing-2'
+
+      - name: commit changes
+        run: |
+          git config user.name "github-workflow"
+          git config user.email "github-workflow"
+          git add --all
+          git commit -m "Run of functional testing" || echo "No changes to commit"
+          git push
+        working-directory: 'functional-testing-2'

.github/workflows/style-test.yml

@@ -12,6 +12,7 @@ on:
 jobs:
 
   build-and-test:
+    if: github.event.pull_request.draft == false
     name: Style
     runs-on: ubuntu-latest
 

.github/workflows/unit-test-os-coverage.yml

@@ -12,6 +12,7 @@ on:
 jobs:
 
   build-and-test:
+    if: github.event.pull_request.draft == false
     name: OS unit tests
     runs-on: ${{ matrix.os }}
 

.github/workflows/unit-test-python-coverage.yml

@@ -12,6 +12,7 @@ on:
 jobs:
 
   build-and-test:
+    if: github.event.pull_request.draft == false
     name: Python unit tests
     runs-on: ubuntu-latest
 

pints/functionaltests/__init__.py

@@ -0,0 +1,7 @@
+#
+# This file is part of PINTS (https://github.com/pints-team/pints/) which is
+# released under the BSD 3-clause license. See accompanying LICENSE.md for
+# copyright notice and full license details.
+#
+
+from .functest_haario_bardenet_acmc import TestHaarioBardenetACMCOn2dimGaussianDistribution

pints/functionaltests/functest_haario_bardenet_acmc.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+#
+# This file is part of PINTS (https://github.com/pints-team/pints/) which is
+# released under the BSD 3-clause license. See accompanying LICENSE.md for
+# copyright notice and full license details.
+#
+
+from __future__ import division
+
+import numpy as np
+
+import pints
+import pints.toy
+
+
+class TestHaarioBardenetACMCOn2dimGaussianDistribution(object):
+    """
+    Tests the pints.HaarioBardenetACMC on a standard 2 dimensional Gaussian
+    distribution.
+    """
+    def __init__(self):
+        # Define calibrated test parameters
+        n_chains = 3
+        n_iterations = 4000
+        warmup = 1000
+        method = pints.HaarioBardenetACMC
+
+        # Define pdf
+        self.pdf = pints.toy.GaussianLogPDF(mean=[0, 0], sigma=[1, 1])
+
+        # Get initial parameters
+        log_prior = pints.ComposedLogPrior(
+            pints.GaussianLogPrior(mean=0, sd=100),
+            pints.GaussianLogPrior(mean=0, sd=100))
+        initial_parameters = log_prior.sample(n=n_chains)
+
+        # Set up sampler
+        sampler = pints.MCMCController(
+            self.pdf, n_chains, initial_parameters, method=method)
+        sampler.set_max_iterations(n_iterations)
+        sampler.set_log_to_screen(False)
+
+        # Infer posterior and throw away warm-up
+        chains = sampler.run()
+        self.chains = chains[:, warmup:]
+
+    def estimate_kld(self):
+        """
+        Estimates and returns the Kullback-Leibler divergence between the
+        approximate posterior and the true posterior assuming that the
+        approximated posterior has Gaussian shape.
+        """
+        # Pool samples from chains
+        chains_x = self.chains[:, :, 0].flatten()
+        chains_y = self.chains[:, :, 1].flatten()
+        chains = np.vstack([chains_x, chains_y]).T
+
+        return self.pdf.kl_divergence(chains)
+
+    def estimate_mean_ess(self):
+        """
+        Estimates the effective sample size (ESS) for each chain and each
+        parameter and returns the mean ESS for across all chains and
+        parameters.
+        """
+        # Estiomate mean ESS for each chain
+        n_chains, _, n_parameters = self.chains.shape
+        ess = np.empty(shape=(n_chains, n_parameters))
+        for chain_id, chain in enumerate(self.chains):
+            ess[chain_id] = pints.effective_sample_size(chain)
+
+        return np.mean(ess)
+
+    def get_results(self):
+        """
+        Runs the functional tests and returns the results.
+        """
+        results = {}
+
+        # Estimate
+        results['kld'] = self.estimate_kld()
+        results['mean-ess'] = self.estimate_mean_ess()
+
+        return results
+
+    @staticmethod
+    def get_name():
+        """
+        Returns the name of this class.
+        """
+        return 'TestHaarioBardenetACMCOn2dimGaussianDistribution'