Merge pull request #12 from cdepillabout/new-package-set

Add method for creating completely new package set

Author: cdepillabout

.github/workflows/ci.yml

@@ -102,6 +102,10 @@ jobs:
         run: "nix develop -L --command cabal test all"
         working-directory: "./my-example-haskell-lib-advanced/"
 
+      - name: "Tests"
+        run: "nix-build"
+        working-directory: "./test/"
+
   nix-commit-from-flake-lock:
     name: nix / ubuntu-latest / using stacklock2nix commit from flake.lock
     runs-on: ubuntu-latest

.gitignore

@@ -1,6 +1,7 @@
 
 # Nix result files
-/result
+result
+result-*
 
 # Haskell .gitignore
 dist

CHANGELOG.md

@@ -1,3 +1,32 @@
+## 1.1.0
+
+*   Added two new attributes to the attribute set returned from a call to
+    `stacklock2nix`: `newPkgSet` and `newPkgSetDevShell`.  These two values are
+    similar to the existing `pkgSet` and `devShell` attributes.  Whereas
+    `pkgSet` and `devShell` take the `baseHaskellPkgSet` argument and overlay
+    it with package overrides created from your `stack.yaml` file, `newPkgSet`
+    and `newPkgSetDevShell` are a completely new package set, containing _only_
+    packages from your `stack.yaml`.
+
+    The effect of this is that `pkgSet` will contain packages that are in
+    Nixpkgs, but not in Stackage.  For instance, when using `pkgSet`, you
+    should be able to access the package
+    [`pkgSet.termonad`](https://hackage.haskell.org/package/termonad) because
+    it is available on Hackage (and in Nixpkgs), even though it is not in any
+    Stackage resolver.
+
+    However, `newPkgSet` will only contain packages in your `stack.yaml` file.
+    For instance, you'll never be able to access `newPkgSet.termonad` or
+    `newPkgSet.spago`, because they will likely never be available on Stackage.
+
+    In general, in your own projects, should you use `pkgSet` or `newPkgSet`?
+
+    For building your own projects, most of the time `pkgSet` and `newPkgSet`
+    should be similar.  `newPkgSet` may be slightly safer, since there is
+    almost no chance you accidentally use a Haskell package outside of your
+    `stack.yaml`.  `pkgSet` may be slightly more convenient depending on what
+    you're trying to do.
+
 ## 1.0.0
 
 *   This is the 1.0 release of `stacklock2nix`.  I've tested `stacklock2nix` on

nix/build-support/stacklock2nix/cabal2nixArgsForPkg.nix

@@ -36,24 +36,55 @@
 #     for testing, but this is not necessary to build the Haskell library.
 #
 # Please feel free to send PRs adding necessary overrides here.
+#
+# Make sure to keep this list in alphabetical order.
 
 cabal2nixArgsOverrides {
   "gi-cairo" = ver: { cairo = pkgs.cairo; };
+
   "gi-gdk" = ver: { gtk3 = pkgs.gtk3; };
+
   "gi-gio" = ver: { glib = pkgs.glib; };
+
   "gi-glib" = ver: { glib = pkgs.glib; };
+
   "gi-gtk" = ver: { gtk3 = pkgs.gtk3; };
+
   "gi-gmodule" = ver: { gmodule = null; };
+
   "gi-gobject" = ver: { glib = pkgs.glib; };
+
   "gi-harfbuzz" = ver: { harfbuzz-gobject = null; };
+
   "gi-pango" = ver: { cairo = pkgs.cairo; pango = pkgs.pango; };
+
   "gi-vte" = ver: { vte_291 = pkgs.vte; };
+
   "glib" = ver: { glib = pkgs.glib; };
+
   "haskell-gi" = ver: { glib = pkgs.glib; gobject-introspection = pkgs.gobject-introspection; };
+
   "haskell-gi-base" = ver: { glib = pkgs.glib; };
+
+  # The PSQueue and fingertree-psqueue packages are used in benchmarks, but they are not on Stackage.
+  "psqueues" = ver: { fingertree-psqueue = null; PSQueue = null; };
+
   "saltine" = ver: { libsodium = pkgs.libsodium; };
+
   "secp256k1-haskell" = ver: { secp256k1 = pkgs.secp256k1; };
+
   "splitmix" = ver: { testu01 = null; };
+
+  "test-framework" = ver: { libxml = pkgs.libxml; };
+
   "termonad" = ver: { vte_291 = pkgs.vte; gtk3 = pkgs.gtk3; pcre2 = pkgs.pcre2;};
+
+  # unordered-containers uses the Haskell package nothunks in its test-suite,
+  # but nothunks is not in Stackage.  We disable the tests for unordered-containers
+  # in the suggestedOverlay.nix file, but callCabal2Nix is called on it before
+  # suggestedOverlay.nix is applied.  So here we need to just pass in null for
+  # the nothunks dependency, since it won't end up being used.
+  "unordered-containers" = ver: { nothunks = null; };
+
   "zlib" = ver: { zlib = pkgs.zlib; };
 }

nix/build-support/stacklock2nix/default.nix

@@ -5,6 +5,7 @@
 , lib
 , runCommand
 , stdenv
+, path
 }@topargs:
 
 { # The path to your stack.yaml file.
@@ -77,6 +78,10 @@
   # This is not used if `baseHaskellPkgSet` is `null`.
   all-cabal-hashes ? null
 , callPackage ? topargs.callPackage
+, # Path to Nixpkgs.
+  #
+  # nixpkgsPath :: Path
+  nixpkgsPath ? topargs.path
 }:
 
 # The stack.yaml path can be computed from the stack.yaml.lock path, or
@@ -372,7 +377,7 @@ let
   # This roughly returns an overlay that looks like the following:
   # ```
   # { lens = callHackage "lens" "5.0.1" {};
-  #   conduit = callHackage "conduit" "5.0.1" {};
+  #   conduit = callHackage "conduit" "1.3.4" {};
   # }
   # ```
   haskPkgLocksToOverlay = haskPkgLocks: hfinal: hprev:
@@ -635,22 +640,135 @@ let
         inherit all-cabal-hashes;
       });
 
+  devShellForPkgSet = packageSet:
+    if packageSet == null then
+      null
+    else
+      packageSet.shellFor {
+        packages = localPkgsSelector;
+        nativeBuildInputs = additionalDevShellNativeBuildInputs packageSet;
+      };
+
   # A development shell created by passing all your local packages (from
   # `localPkgsSelector`) to `pkgSet.shellFor`.
   #
   # devShell :: Drv
   #
   # Note that this derivation is specifically meant to be passed to `nix
   # develop` or `nix-shell`.
-  devShell =
-    if pkgSet == null then
+  devShell = devShellForPkgSet pkgSet;
+
+  # An Nixpkgs Haskell overlay that has GHC boot packages set to `null`. This
+  # is used as an initial overlay when creating a brand new package set.
+  newPkgSetCompilerConfig = self: super: {
+    # TODO: Should llvmPackages be enabled here?
+    # llvmPackages = pkgs.lib.dontRecurseIntoAttrs self.ghc.llvmPackages;
+
+    # Disable GHC core libraries.
+    array = null;
+    base = null;
+    binary = null;
+    bytestring = null;
+    Cabal = null;
+    containers = null;
+    deepseq = null;
+    directory = null;
+    exceptions = null;
+    filepath = null;
+    ghc-bignum = null;
+    ghc-boot = null;
+    ghc-boot-th = null;
+    ghc-compact = null;
+    ghc-heap = null;
+    ghc-prim = null;
+    ghci = null;
+    haskeline = null;
+    hpc = null;
+    integer-gmp = null;
+    libiserv = null;
+    mtl = null;
+    parsec = null;
+    pretty = null;
+    process = null;
+    rts = null;
+    stm = null;
+    template-haskell = null;
+
+    # GHC only builds terminfo if it is a native compiler
+    # terminfo = if pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform then null else self.terminfo_0_4_1_5;
+    terminfo = null;
+
+    text = null;
+    time = null;
+    transformers = null;
+    unix = null;
+
+    # GHC only bundles the xhtml library if haddock is enabled, check if this is
+    # still the case when updating: https://gitlab.haskell.org/ghc/ghc/-/blob/0198841877f6f04269d6050892b98b5c3807ce4c/ghc.mk#L463
+    # xhtml = if self.ghc.hasHaddock or true then null else self.xhtml_3000_2_2_1;
+    xhtml = null;
+  };
+
+  # This is similar to `pkgSet`.
+  #
+  # While `pkgSet` is `baseHaskellPkgSet` overridden with overlays from your
+  # stack.yaml.lock file, `newPkgSet` is a completely new Nixpkgs Haskell
+  # package set.  It _only_ contains packages defined in the `stack.yaml` file.
+  #
+  # newPkgSet :: HaskellPkgSet
+  #
+  # `newPkgSet` will contain local packages.  For instance, if you have a local
+  # package called `my-haskell-pkg`:
+  #
+  # Example: `newPkgSet.my-haskell-pkg`
+  #
+  # `newPkgSet` will also contain packages in your stack.yaml resolver. For
+  # instance:
+  #
+  # Example: `newPkgSet.lens`
+  #
+  # `pkgSet` will _not_ contain packages from the underlying Haskell package
+  # set.  For instance, `termonad` is not available in Stackage, so it is not
+  # available in `newPkgSet`.
+  newPkgSet =
+    if baseHaskellPkgSet == null then
       null
     else
-      pkgSet.shellFor {
-        packages = localPkgsSelector;
-        nativeBuildInputs = additionalDevShellNativeBuildInputs pkgSet;
-      };
+      let
+        haskPkgSet = callPackage (nixpkgsPath + "/pkgs/development/haskell-modules") {
+          haskellLib = haskell.lib.compose;
+
+          # TODO: Is it okay to use a completely different package set as the
+          # base package set like this?
+          buildHaskellPackages = baseHaskellPkgSet;
+
+          ghc = baseHaskellPkgSet.ghc;
+
+          compilerConfig = newPkgSetCompilerConfig;
+
+          initialPackages = _: _: {};
+
+          overrides = lib.composeManyExtensions [
+            # It is not possible to put these overlays into the
+            # `initialPackages` argument, because they use functions like
+            # `callHackage` and `callCabal2nix`, which appear to not be
+            # available when `initialPackages` gets evaluated.
+            combinedOverlay
+            additionalHaskellPkgSetOverrides
+          ];
+
+          nonHackagePackages = _: _: {};
+          configurationCommon = _: _: _: {};
+          configurationNix = _: _: _: {};
+          configurationArm = _: _: _: {};
+          configurationDarwin = _: _: _: {};
+        };
+      in haskPkgSet;
 
+  # Same as `devShell`, but based on `newPkgSet`.
+  #
+  # newPkgSetDevShell :: Drv
+  newPkgSetDevShell = devShellForPkgSet newPkgSet;
 in
 
 { inherit
@@ -662,6 +780,8 @@ in
     localPkgsSelector
     pkgSet
     devShell
+    newPkgSet
+    newPkgSetDevShell
     ;
 
   # These are a bunch of internal attributes, used for testing.

nix/build-support/stacklock2nix/suggestedOverlay.nix

@@ -28,6 +28,8 @@
 # ```
 #
 # These will be maintained on a best-effort basis.  Again, please send PRs.
+#
+# Make sure to keep this list in alphabetical order.
 
 hfinal: hprev: with haskell.lib.compose; {
 
@@ -155,9 +157,6 @@ hfinal: hprev: with haskell.lib.compose; {
 
   nanospec = dontCheck hprev.nanospec;
 
-  # test suite doesn't build
-  nothunks = dontCheck hprev.nothunks;
-
   # circular dependency in tests
   options = dontCheck hprev.options;
 
@@ -213,6 +212,9 @@ hfinal: hprev: with haskell.lib.compose; {
   # tests don't support musl
   unix-time = dontCheck hprev.unix-time;
 
+  # Test suite requires the nothunks library, which isn't on stackage.
+  unordered-containers = dontCheck hprev.unordered-containers;
+
   vector = dontCheck hprev.vector;
 
   # test suite uses phantom js

test/default.nix

@@ -0,0 +1,4 @@
+
+{...}:
+
+(import ./nixpkgs.nix {}).stacklock2nix-tests

test/nixpkgs.nix

@@ -0,0 +1,30 @@
+
+{...}:
+
+let
+  flake-lock = builtins.fromJSON (builtins.readFile ../my-example-haskell-lib-advanced/flake.lock);
+
+  # Use the Nixpkgs that the my-example-haskell-lib-advanced is pinned to.
+  nixpkgs-src = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${flake-lock.nodes.nixpkgs.locked.rev}.tar.gz";
+    sha256 = flake-lock.nodes.nixpkgs.locked.narHash;
+  };
+
+  overlays = [
+    # stacklock2nix overlay
+    (import ../nix/overlay.nix)
+
+    # tests
+    (final: prev: {
+      stacklock2nix-tests =
+        (final.callPackage ./test-new-package-set.nix {}) ++ [
+          # new tests go here
+        ];
+    })
+  ];
+
+  pkgs = import nixpkgs-src { inherit overlays; };
+
+in
+
+pkgs

test/test-new-package-set.nix

@@ -0,0 +1,43 @@
+
+{ stacklock2nix
+, haskell
+, cabal-install
+, fetchurl
+}:
+
+let
+  hasklib = haskell.lib.compose;
+
+  stacklock = stacklock2nix {
+    stackYaml = ../my-example-haskell-lib-advanced/stack.yaml;
+    baseHaskellPkgSet = haskell.packages.ghc924;
+    additionalHaskellPkgSetOverrides = hfinal: hprev: {
+      # The servant-cassava.cabal file is malformed on GitHub:
+      # https://github.com/haskell-servant/servant-cassava/pull/29
+      servant-cassava =
+        hasklib.overrideCabal
+          { editedCabalFile = null; revision = null; }
+          hprev.servant-cassava;
+
+      amazonka = hasklib.dontCheck hprev.amazonka;
+      amazonka-core = hasklib.dontCheck hprev.amazonka-core;
+      amazonka-sso = hasklib.dontCheck hprev.amazonka-sso;
+      amazonka-sts = hasklib.dontCheck hprev.amazonka-sts;
+    };
+    cabal2nixArgsOverrides = args: args // {
+      amazonka-sso = ver: { amazonka-test = null; };
+      amazonka-sts = ver: { amazonka-test = null; };
+    };
+    additionalDevShellNativeBuildInputs = stacklockHaskellPkgSet: [
+      cabal-install
+    ];
+    all-cabal-hashes = fetchurl {
+      name = "all-cabal-hashes";
+      url = "https://github.com/commercialhaskell/all-cabal-hashes/archive/9ab160f48cb535719783bc43c0fbf33e6d52fa99.tar.gz";
+      sha256 = "sha256-QC07T3MEm9LIMRpxIq3Pnqul60r7FpAdope6S62sEX8=";
+    };
+  };
+in
+[ stacklock.newPkgSet.my-example-haskell-app
+  stacklock.newPkgSetDevShell
+]