Merge fort-nix/nix-bitcoin#379: Add flake support

dde04f8cbe update nixpkgs-unstable (Erik Arvstedt)
87df809a88 add helper/update-flake.sh (Erik Arvstedt)
f7c2133250 add flake support (Erik Arvstedt)
de77281cba pkgs: import pinned nixpkgs in default.nix (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  nixbitcoin:
    ACK dde04f8cbe
  jonasnick:
    ACK dde04f8cbe

Tree-SHA512: 251bbaf748bc429e1f094bf37f1bbf00eed91108f165e378f3ac4de817d42f3e1e4e8c1bbf79ea81447f24d0c172e81fd4eec98164379ea27095d6ba5420af6c

examples/README.md

@@ -69,9 +69,17 @@ c systemctl status bitcoind
 ```
 See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation.
 
-
 ### Real-world example
 Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org
 to see the configuration of a nix-bitcoin node that's used in production.
 
 The commands in `shell.nix` allow you to locally run the node in a VM or container.
+
+### Flakes
+
+Flakes make it easy to include `nix-bitcoin` in an existing NixOS config.
+The [flakes example](./flakes/flake.nix) shows how to use `nix-bitcoin` as an input to a system flake.
+
+Run `nix run` or `nix run .#vm` from the nix-bitcoin root directory to start an example
+nix-bitcoin node VM.
+This command is defined by the nix-bitcoin flake (in [flake.nix](../flake.nix)).

examples/flakes/flake.nix

@@ -0,0 +1,49 @@
+{
+  description = "A basic nix-bitcoin node";
+
+  inputs.nix-bitcoin.url = "github:fort-nix/nix-bitcoin";
+
+  outputs = { self, nix-bitcoin }: {
+
+    nixosConfigurations.mynode = nix-bitcoin.inputs.nixpkgs.lib.nixosSystem {
+      system = "x86_64-linux";
+      modules = [
+        ## Note:
+        ## If you use a custom nixpkgs version for evaluating your system,
+        ## consider using `withLockedPkgs` instead of `withSystemPkgs` to use the exact
+        ## pkgs versions for nix-bitcoin services that are tested by nix-bitcoin.
+        ## The downsides are increased evaluation times and increased system
+        ## closure size.
+        #
+        # nix-bitcoin.nixosModules.withLockedPkgs
+        nix-bitcoin.nixosModules.withSystemPkgs
+
+        ## Optional:
+        ## Import the secure-node preset, an opinionated config to enhance security
+        ## and privacy.
+        #
+        # "${nix-bitcoin}/modules/presets/secure-node.nix"
+
+        {
+          nix-bitcoin.generateSecrets = true;
+
+          services.bitcoind.enable = true;
+
+          # When using nix-bitcoin as part of a larger NixOS configuration, set the following to enable
+          # interactive access to nix-bitcoin features (like bitcoin-cli) for your system's main user
+          nix-bitcoin.operator = {
+            enable = true;
+            name = "main"; # Set this to your system's main user
+          };
+
+          # The system's main unprivileged user. This setting is usually part of your
+          # existing NixOS configuration.
+          users.users.main = {
+            isNormalUser = true;
+            password = "a";
+          };
+        }
+      ];
+    };
+  };
+}

flake.lock

@@ -0,0 +1,60 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1623875721,
+        "narHash": "sha256-A8BU7bjS5GirpAUv4QA+QnJ4CceLHkcXdRp4xITDB0s=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "f7e004a55b120c02ecb6219596820fcd32ca8772",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1628865210,
+        "narHash": "sha256-dB3IA8AYUQDXH+Xy/6nbv4QpIbVl88DphbcxJSUYiX4=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a445f5829889959d65ad65e5c961d5c67e1cd677",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-21.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgsUnstable": {
+      "locked": {
+        "lastModified": 1628934079,
+        "narHash": "sha256-CEYsKXNYprs/TvmB7ppkYMALXnfhEw6lg5VaEXgpoec=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "fd9984fd9a950686e7271ecf01893987a42cdf14",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs",
+        "nixpkgsUnstable": "nixpkgsUnstable"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,106 @@
+{
+  description = ''
+    A collection of Nix packages and NixOS modules for easily
+    installing full-featured Bitcoin nodes with an emphasis on security.
+  '';
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.05";
+    nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, nixpkgsUnstable, flake-utils }:
+    let
+      supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ];
+    in
+    rec {
+      mkNbPkgs = {
+        system
+        , pkgs ? import nixpkgs { inherit system; }
+        , pkgsUnstable ? import nixpkgsUnstable { inherit system; }
+      }:
+        import ./pkgs { inherit pkgs pkgsUnstable; };
+
+      overlay = final: prev: let
+        nbPkgs = mkNbPkgs { inherit (final) system; pkgs = final; };
+      in removeAttrs nbPkgs [ "pinned" "nixops19_09" "krops" ];
+
+      nixosModules = {
+        # Uses the default system pkgs for nix-bitcoin.pkgs
+        withSystemPkgs =  { pkgs, ... }: {
+          imports = [ ./modules/modules.nix ];
+          nix-bitcoin.pkgs = (mkNbPkgs { inherit (pkgs) system; inherit pkgs; }).modulesPkgs;
+        };
+
+        # Uses the nixpkgs version locked by this flake for nix-bitcoin.pkgs.
+        # More stable, but slightly slower to evaluate and needs more space if the
+        # locked and the system nixpkgs versions differ.
+        withLockedPkgs =  { config, ... }: {
+          imports = [ ./modules/modules.nix ];
+          nix-bitcoin.pkgs = (mkNbPkgs { inherit (config.nixpkgs) system; }).modulesPkgs;
+        };
+      };
+
+      defaultTemplate = {
+        description = "Basic node template";
+        path = ./examples/flakes;
+      };
+
+    } // (flake-utils.lib.eachSystem supportedSystems (system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        mkVMScript = vm: pkgs.writers.writeBash "run-vm" ''
+          set -euo pipefail
+          export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
+          trap "rm -rf $TMPDIR" EXIT
+          export NIX_DISK_IMAGE=$TMPDIR/nixos.qcow2
+          QEMU_OPTS="-smp $(nproc) -m 1500" ${vm}/bin/run-*-vm
+        '';
+      in rec {
+        nbPkgs = self.mkNbPkgs { inherit system pkgs; };
+
+        packages = flake-utils.lib.flattenTree (removeAttrs nbPkgs [
+          "pinned" "modulesPkgs" "nixops19_09" "krops"
+        ]) // {
+          runVM = mkVMScript packages.vm;
+
+          # This is a simple demo VM.
+          # See ./examples/flakes/flake.nix on how to use nix-bitcoin with flakes.
+          vm = let
+            nix-bitcoin = self;
+          in
+            (import "${nixpkgs}/nixos" {
+              inherit system;
+              configuration = {
+                imports = [
+                  nix-bitcoin.nixosModules.withSystemPkgs
+                  "${nix-bitcoin}/modules/presets/secure-node.nix"
+                ];
+
+                nix-bitcoin.generateSecrets = true;
+                services.clightning.enable = true;
+                # For faster startup in offline VMs
+                services.clightning.extraConfig = "disable-dns";
+
+                nixpkgs.pkgs = pkgs;
+                virtualisation.graphics = false;
+                services.getty.autologinUser = "root";
+                nix.nixPath = [ "nixpkgs=${nixpkgs}" ];
+              };
+            }).vm;
+        };
+
+        defaultApp = apps.vm;
+
+        apps = {
+          # Run a basic nix-bitcoin node in a VM
+          vm = {
+            type = "app";
+            program = toString packages.runVM;
+          };
+        };
+      }
+    ));
+}

helper/update-flake.nix

@@ -0,0 +1,77 @@
+{ prevVersions ? null }:
+let
+  flake = builtins.getFlake (toString ../.);
+  inherit (flake.inputs.nixpkgs) lib;
+in rec {
+  # Convert the list of pinned pkgs to an attrset with form
+  # {
+  #   stable = { bitcoind = "0.21.1"; ... };
+  #   unstable = { btcpayserver = "1.2.1"; ... };
+  # }
+  # A pinned pkg is added to `stable` if the stable and unstable pkg versions
+  # are identical.
+  versions = let
+    pinned = flake.nbPkgs.x86_64-linux.pinned;
+    pinnedPkgs = lib.filterAttrs (n: v: lib.isDerivation v) pinned;
+    stable = pinned.pkgs;
+    unstable = pinned.pkgsUnstable;
+    isStable = builtins.partition (pkgName:
+      !(unstable ? "${pkgName}") ||
+      ((stable ? "${pkgName}") && stable.${pkgName}.version == unstable.${pkgName}.version)
+    ) (builtins.attrNames pinnedPkgs);
+  in {
+    stable   = lib.genAttrs isStable.right (pkgName: stable.${pkgName}.version);
+    unstable = lib.genAttrs isStable.wrong (pkgName: unstable.${pkgName}.version);
+  };
+
+  showUpdates = let
+    prev = builtins.fromJSON prevVersions;
+    prevPkgs = prev.stable // prev.unstable;
+    mapFilter = f: xs: lib.remove null (map f xs);
+
+    mkChanges = title: pkgs:
+      let
+        lines = mapFilter (pkgName:
+          let
+            version = pkgs.${pkgName};
+            prevVersion = prevPkgs.${pkgName};
+          in
+            if version != prevVersion then
+              "${pkgName}: ${prevVersion} -> ${version}"
+            else
+              null
+        ) (builtins.attrNames pkgs);
+      in
+        if lines == [] then
+             null
+        else
+          builtins.concatStringsSep "\n" ([title] ++ lines);
+
+    changes = [
+      (mkChanges "Pkg updates in nixpkgs stable:" versions.stable)
+      (mkChanges "Pkg updates in nixpkgs unstable:" versions.unstable)
+    ];
+
+    allChanges = builtins.concatStringsSep "\n\n" (lib.remove null changes);
+  in
+    if allChanges == "" then
+      "No pkg version updates."
+    else
+      allChanges;
+
+  pinnedFile = let
+    toLines = pkgs: builtins.concatStringsSep "\n    " (builtins.attrNames pkgs);
+  in ''
+    # This file is generated by ../helper/update-flake.nix
+    pkgs: pkgsUnstable:
+    {
+      inherit (pkgs)
+        ${toLines versions.stable};
+
+      inherit (pkgsUnstable)
+        ${toLines versions.unstable};
+
+      inherit pkgs pkgsUnstable;
+    }
+  '';
+}

helper/update-flake.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script does the following:
+# - Update all flake inputs, including nixpkgs
+# - Print version updates of pinned pkgs like so:
+#     Pkg updates in nixpkgs unstable:
+#     bitcoin: 0.20.0 -> 0.21.1
+#     btcpayserver: 1.1.0 -> 1.1.2
+# - Write ../pkgs/pinned.nix:
+#     Packages for which the stable und unstable versions are identical are
+#     pinned to stable.
+#     All other pkgs are pinned to unstable.
+
+# cd to script dir
+cd "${BASH_SOURCE[0]%/*}"
+
+if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then
+    echo "Error. This script requires nix flake support."
+    echo "https://nixos.wiki/wiki/Flakes#Installing_flakes"
+    exit 1
+fi
+
+if [[ ${1:-} != -f ]] && ! git diff --quiet ../flake.{nix,lock}; then
+    echo "error: flake.nix/flake.lock have changes. Run with option -f to ignore."
+    exit 1
+fi
+
+versions=$(nix eval --json -f update-flake.nix versions)
+
+## Uncomment the following to generate a version change message for testing
+# versions=$(echo "$versions" | sed 's|1|0|g')
+
+nix flake update ..
+
+echo
+nix eval --raw -f update-flake.nix --argstr prevVersions "$versions" showUpdates; echo
+
+pinned=../pkgs/pinned.nix
+pinnedSrc=$(nix eval --raw -f update-flake.nix --argstr prevVersions "$versions" pinnedFile)
+if [[ $pinnedSrc != $(cat "$pinned") ]]; then
+    echo "$pinnedSrc" > "$pinned"
+    echo
+    echo "Updated pinned.nix"
+fi

pkgs/default.nix

@@ -1,4 +1,10 @@
-{ pkgs ? import <nixpkgs> {} }:
+let
+  nixpkgsPinned = import ./nixpkgs-pinned.nix;
+in
+# Set default values for use without flakes
+{ pkgs ? import <nixpkgs> { config = {}; overlays = []; }
+, pkgsUnstable ? import nixpkgsPinned.nixpkgs-unstable { config = {}; overlays = []; }
+}:
 let self = {
   spark-wallet = pkgs.callPackage ./spark-wallet { };
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
@@ -8,7 +14,7 @@ let self = {
   krops = import ./krops { };
   netns-exec = pkgs.callPackage ./netns-exec { };
   extra-container = pkgs.callPackage ./extra-container { };
-  clightning-plugins = import ./clightning-plugins pkgs self.nbPython3Packages;
+  clightning-plugins = pkgs.recurseIntoAttrs (import ./clightning-plugins pkgs self.nbPython3Packages);
   clboss = pkgs.callPackage ./clboss { };
   secp256k1 = pkgs.callPackage ./secp256k1 { };
 
@@ -16,7 +22,7 @@ let self = {
     packageOverrides = import ./python-packages self;
   }).pkgs;
 
-  pinned = import ./pinned.nix;
+  pinned = import ./pinned.nix pkgs pkgsUnstable;
 
   modulesPkgs = self // self.pinned;
 }; in self

pkgs/nixpkgs-pinned.nix

@@ -1,19 +1,20 @@
 let
-  fetch = { rev, sha256 }:
+  fetchNixpkgs = { rev, sha256 }:
     builtins.fetchTarball {
       url = "https://github.com/nixos/nixpkgs/archive/${rev}.tar.gz";
       inherit sha256;
     };
+
+  fetch = input: let
+    inherit (input) locked;
+  in fetchNixpkgs {
+    inherit (locked) rev;
+    sha256 = locked.narHash;
+  };
+
+  lockedInputs = (builtins.fromJSON (builtins.readFile ../flake.lock)).nodes;
 in
 {
-  # To update, run ../helper/fetch-channel REV
-  nixpkgs = fetch {
-    # nixos-21.05 (2021-08-14)
-    rev = "a445f5829889959d65ad65e5c961d5c67e1cd677";
-    sha256 = "0zl930jjacdphplw1wv5nlhjk15zvflzzwp53zbh0l8qq01wh7bl";
-  };
-  nixpkgs-unstable = fetch {
-    rev = "4138cbd913fad85073e59007710e3f083d0eb7c6";
-    sha256 = "0l7vaa6mnnmxfxzi9i5gd4c4j3cpfh7gjsjsfk6nnj1r05pazf0j";
-  };
+  nixpkgs = fetch lockedInputs.nixpkgs;
+  nixpkgs-unstable = fetch lockedInputs.nixpkgsUnstable;
 }

pkgs/pinned.nix

@@ -1,28 +1,22 @@
-let
-  nixpkgsPinned = import ./nixpkgs-pinned.nix;
-  nixpkgsStable = import nixpkgsPinned.nixpkgs { config = {}; overlays = []; };
-  nixpkgsUnstable = import nixpkgsPinned.nixpkgs-unstable { config = {}; overlays = []; };
-  nixBitcoinPkgsStable = import ./. { pkgs = nixpkgsStable; };
-  nixBitcoinPkgsUnstable = import ./. { pkgs = nixpkgsUnstable; };
-in
+# This file is generated by ../helper/update-flake.nix
+pkgs: pkgsUnstable:
 {
-  inherit (nixpkgsUnstable)
+  inherit (pkgs)
     bitcoin
     bitcoind
+    lndconnect;
+
+  inherit (pkgsUnstable)
+    btcpayserver
     charge-lnd
     clightning
-    lnd
-    lndconnect
-    nbxplorer
-    btcpayserver
     electrs
     elementsd
     hwi
     lightning-loop
-    lightning-pool;
+    lightning-pool
+    lnd
+    nbxplorer;
 
-  inherit nixpkgsStable nixpkgsUnstable;
-
-  stable = nixBitcoinPkgsStable;
-  unstable = nixBitcoinPkgsUnstable;
+  inherit pkgs pkgsUnstable;
 }

pkgs/python-packages/default.nix

@@ -4,8 +4,6 @@ let
 
   joinmarketPkg = pkg: callPackage pkg { inherit (nbPkgs.joinmarket) version src; };
   clightningPkg = pkg: callPackage pkg { inherit (nbPkgs.pinned) clightning; };
-
-  unstable = (import ../nixpkgs-pinned.nix).nixpkgs-unstable;
 in {
   bencoderpyx = callPackage ./bencoderpyx {};
   coincurve = callPackage ./coincurve {};

test/run-tests.sh

@@ -281,9 +281,18 @@ examples() {
     (cd "$scriptDir/../examples" && nix-shell --run "$script")
 }
 
+flake() {
+    if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then
+        echo "Skipping flake test. Nix flake support is not enabled."
+    else
+        nix flake check "$scriptDir/.."
+    fi
+}
+
 all() {
     buildable
     examples
+    flake
 }
 
 # An alias for buildTest