Merge fort-nix/nix-bitcoin#429: Add nixos-search support

5e59c784ed flake: remove internal pkg `netns-exec` from packages (Erik Arvstedt)
1ef40ac230 ci: add flake test (Erik Arvstedt)
d3c8d7bd6f .cirrus.yml: refactor (Erik Arvstedt)
26a53f0d59 ci: simplify test (Erik Arvstedt)
0fd3be5343 move ci/ to test/ (Erik Arvstedt)
eaf98f92db test: add test `nixosSearch` (Erik Arvstedt)
77d8c00c53 run-tests.sh: extract `checkFlakeSupport` (Erik Arvstedt)
96df81a4d0 run-tests: move `flake` next to other tests (Erik Arvstedt)
bbebd0b383 treewide: remove invalid docbook XML from option descriptions (Erik Arvstedt)
3681f118f7 nix-bitcoin.nix: add `defaultText` (automatic) (Erik Arvstedt)
d24c029435 treewide: add `defaultText` (manual) (Erik Arvstedt)
f0096371bf btcpayserver: expand `nbPkgs` in option defaults (Erik Arvstedt)
349f904e79 secrets: use assertion instead of error in default value (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  nixbitcoin:
    reACK 5e59c784ed

Tree-SHA512: affecb3acd0cc5191c8225fd61a3c0869620c307fbdf02683637d1aa6ef4ea889b420496639046391e23ae2506a9cb2d9d2e6cfc0cb281e8c20f00d5508fc644

.cirrus.yml

@@ -1,37 +1,38 @@
-container:
-  image: nixos/nix
-  # Besides virtualization, this also enables privileged containers which are required for
-  # sandboxed builds
-  kvm: true
-  # Needed for package builds
-  memory: 8G
-
-environment:
-  CACHIX_SIGNING_KEY: ENCRYPTED[!cec502ed813cbcd0237697d2031f750186ff20eed5815b1ad950ad2f2d701702ae6ba2f0cb4cb1985687a696c8ee492c!]
-  # Save some traffic by excluding the full git history
-  CIRRUS_CLONE_DEPTH: 1
-
 task:
+  environment:
+    CACHIX_SIGNING_KEY: ENCRYPTED[!cec502ed813cbcd0237697d2031f750186ff20eed5815b1ad950ad2f2d701702ae6ba2f0cb4cb1985687a696c8ee492c!]
+    # Save some traffic by excluding the full git history
+    CIRRUS_CLONE_DEPTH: 1
+
   # Use the maximum timeout. Needed when rebuilding packages on a channel update.
   timeout_in: 120m
 
-  name: modules_test
-  environment:
-    nixpkgs: nixpkgs
-  container:
-    # A maximum of 16 CPUs is shared among all concurrent tasks.
-    # https://cirrus-ci.org/faq/#are-there-any-limits
-    cpu: 4
   matrix:
-    - environment:
-        scenario: default
-    - environment:
-        scenario: netns
-    - environment:
-        scenario: netnsRegtest
+    - name: modules_test
+      container:
+        image: nixos/nix
+        # Besides virtualization, this also enables privileged containers which are required for
+        # sandboxed builds
+        kvm: true
+        # Needed for package builds
+        memory: 8G
+        # A maximum of 16 CPUs is shared among all concurrent tasks.
+        # https://cirrus-ci.org/faq/#are-there-any-limits
+        cpu: 4
+      environment:
+        matrix:
+          - scenario: default
+          - scenario: netns
+          - scenario: netnsRegtest
+      # This script is run as root
+      build_script:
+        - echo "sandbox = true" >> /etc/nix/nix.conf
+        - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix nixpkgs)"
+        - nix run -f '<nixpkgs>' bash coreutils cachix -c ./test/ci/build.sh
 
-  # This script is run as root
-  build_script:
-    - echo "sandbox = true" >> /etc/nix/nix.conf
-    - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix $nixpkgs)"
-    - nix run -f '<nixpkgs>' bash coreutils cachix -c ./ci/build.sh
+    - name: flake
+      container:
+        image: nixpkgs/nix-flakes
+      build_script:
+        - nix flake check
+        - ./test/nixos-search/ci-test.sh

flake.nix

@@ -62,7 +62,7 @@
         nbPkgs = self.mkNbPkgs { inherit system pkgs; };
 
         packages = flake-utils.lib.flattenTree (removeAttrs nbPkgs [
-          "pinned" "modulesPkgs" "nixops19_09" "krops" "generate-secrets"
+          "pinned" "modulesPkgs" "nixops19_09" "krops" "generate-secrets" "netns-exec"
         ]) // {
           runVM = mkVMScript packages.vm;
 

helper/update-flake.sh

@@ -26,11 +26,16 @@ if [[ ${1:-} != -f ]] && ! git diff --quiet ../flake.{nix,lock}; then
     exit 1
 fi
 
+echo "Updating flake 'nixos-search'"
+nix flake update ../test/nixos-search
+echo
+
 versions=$(nix eval --json -f update-flake.nix versions)
 
 ## Uncomment the following to generate a version change message for testing
 # versions=$(echo "$versions" | sed 's|1|0|g')
 
+echo "Updating main flake"
 nix flake update ..
 
 echo

modules/bitcoind.nix

@@ -57,6 +57,7 @@ let
       package = mkOption {
         type = types.package;
         default = config.nix-bitcoin.pkgs.bitcoind;
+        defaultText = "config.nix-bitcoin.pkgs.bitcoind";
         description = "The package providing bitcoin binaries.";
       };
       extraConfig = mkOption {
@@ -119,7 +120,7 @@ let
                 example = "f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae";
                 description = ''
                   Password HMAC-SHA-256 for JSON-RPC connections. Must be a string of the
-                  format <SALT-HEX>$<HMAC-HEX>.
+                  format `salt-hex$hmac-hex`.
                 '';
               };
               passwordHMACFromFile = mkOption {
@@ -265,6 +266,7 @@ let
         default = pkgs.writeScriptBin "bitcoin-cli" ''
           exec ${cfg.package}/bin/bitcoin-cli -datadir='${cfg.dataDir}' "$@"
         '';
+        defaultText = "(See source)";
         description = "Binary to connect with the bitcoind instance.";
       };
       tor = nbLib.tor;

modules/btcpayserver.nix

@@ -18,9 +18,10 @@ let
       package = mkOption {
         type = types.package;
         default = if cfg.btcpayserver.lbtc then
-                    nbPkgs.btcpayserver.override { altcoinSupport = true; }
+                    config.nix-bitcoin.pkgs.btcpayserver.override { altcoinSupport = true; }
                   else
-                    nbPkgs.btcpayserver;
+                    config.nix-bitcoin.pkgs.btcpayserver;
+        defaultText = "(See source)";
         description = "The package providing btcpayserver binaries.";
       };
       dataDir = mkOption {
@@ -68,7 +69,8 @@ let
       };
       package = mkOption {
         type = types.package;
-        default = nbPkgs.nbxplorer;
+        default = config.nix-bitcoin.pkgs.nbxplorer;
+        defaultText = "config.nix-bitcoin.pkgs.nbxplorer";
         description = "The package providing nbxplorer binaries.";
       };
       address = mkOption {
@@ -102,7 +104,6 @@ let
 
   cfg = config.services;
   nbLib = config.nix-bitcoin.lib;
-  nbPkgs = config.nix-bitcoin.pkgs;
 
   inherit (config.services) bitcoind liquidd;
 in {

modules/clightning-plugins/clboss.nix

@@ -17,6 +17,7 @@ let cfg = config.services.clightning.plugins.clboss; in
     package = mkOption {
       type = types.package;
       default = config.nix-bitcoin.pkgs.clboss;
+      defaultText = "config.nix-bitcoin.pkgs.clboss";
       description = "The package providing clboss binaries.";
     };
   };

modules/clightning.nix

@@ -69,6 +69,7 @@ let
       default = pkgs.writeScriptBin "lightning-cli" ''
         ${nbPkgs.clightning}/bin/lightning-cli --lightning-dir='${cfg.dataDir}' "$@"
       '';
+      defaultText = "(See source)";
       description = "Binary to connect with the clightning instance.";
     };
     getPublicAddressCmd = mkOption {

modules/joinmarket.nix

@@ -44,11 +44,13 @@ let
     };
     cli = mkOption {
       default = cli;
+      defaultText = "(See source)";
     };
     # Used by ./joinmarket-ob-watcher.nix
     ircServers = mkOption {
       readOnly = true;
       default = ircServers;
+      defaultText = "(See source)";
     };
     # This option is only used by netns-isolation.
     # Tor is always enabled.

modules/lightning-loop.nix

@@ -27,6 +27,7 @@ let
     package = mkOption {
       type = types.package;
       default = config.nix-bitcoin.pkgs.lightning-loop;
+      defaultText = "config.nix-bitcoin.pkgs.lightning-loop";
       description = "The package providing lightning-loop binaries.";
     };
     dataDir = mkOption {
@@ -54,6 +55,7 @@ let
         --macaroonpath '${cfg.dataDir}/${network}/loop.macaroon' \
         --tlscertpath '${secretsDir}/loop-cert' "$@"
       '';
+      defaultText = "(See source)";
       description = "Binary to connect with the lightning-loop instance.";
     };
     tor = nbLib.tor;

modules/lightning-pool.nix

@@ -27,6 +27,7 @@ let
     package = mkOption {
       type = types.package;
       default = config.nix-bitcoin.pkgs.lightning-pool;
+      defaultText = "config.nix-bitcoin.pkgs.lightning-pool";
       description = "The package providing lightning-pool binaries.";
     };
     dataDir = mkOption {
@@ -54,6 +55,7 @@ let
           --network ${network} \
           --basedir '${cfg.dataDir}' "$@"
       '';
+      defaultText = "(See source)";
       description = "Binary to connect with the lightning-pool instance.";
     };
     tor = nbLib.tor;

modules/liquid.nix

@@ -148,12 +148,14 @@ let
         default = pkgs.writeScriptBin "elements-cli" ''
           ${nbPkgs.elementsd}/bin/elements-cli -datadir='${cfg.dataDir}' "$@"
         '';
+        defaultText = "(See source)";
         description = "Binary to connect with the liquidd instance.";
       };
       swapCli = mkOption {
         default = pkgs.writeScriptBin "liquidswap-cli" ''
           ${nbPkgs.liquid-swap}/bin/liquidswap-cli -c '${cfg.dataDir}/elements.conf' "$@"
         '';
+        defaultText = "(See source)";
         description = "Binary for managing liquid swaps.";
       };
       tor = nbLib.tor;
@@ -222,7 +224,7 @@ let
         example = "f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae";
         description = ''
           Password HMAC-SHA-256 for JSON-RPC connections. Must be a string of the
-          format <SALT-HEX>$<HMAC-HEX>.
+          format `salt-hex$hmac-hex`.
         '';
       };
     };

modules/lnd-rest-onion-service.nix

@@ -15,6 +15,7 @@ let
     package = mkOption {
       type = types.package;
       default = config.nix-bitcoin.pkgs.lndconnect;
+      defaultText = "config.nix-bitcoin.pkgs.lndconnect";
       description = "The package providing lndconnect binaries.";
     };
   };

modules/lnd.nix

@@ -81,6 +81,7 @@ let
     package = mkOption {
       type = types.package;
       default = config.nix-bitcoin.pkgs.lnd;
+      defaultText = "config.nix-bitcoin.pkgs.lnd";
       description = "The package providing lnd binaries.";
     };
     cli = mkOption {
@@ -92,6 +93,7 @@ let
             --tlscertpath '${cfg.certPath}' \
             --macaroonpath '${networkDir}/admin.macaroon' "$@"
         '';
+      defaultText = "(See source)";
       description = "Binary to connect with the lnd instance.";
     };
     getPublicAddressCmd = mkOption {

modules/nix-bitcoin.nix

@@ -7,17 +7,20 @@ with lib;
       pkgs = mkOption {
         type = types.attrs;
         default = (import ../pkgs { inherit pkgs; }).modulesPkgs;
+        defaultText = "nix-bitcoin/pkgs.modulesPkgs";
       };
 
       lib = mkOption {
         readOnly = true;
         default = import ../pkgs/lib.nix lib pkgs;
+        defaultText = "nix-bitcoin/pkgs/lib.nix";
       };
 
       torClientAddressWithPort = mkOption {
         readOnly = true;
         default = with config.services.tor.client.socksListenAddress;
           "${addr}:${toString port}";
+        defaultText = "(See source)";
       };
 
       # Torify binary that works with custom Tor SOCKS addresses
@@ -29,6 +32,7 @@ with lib;
             --address ${config.services.tor.client.socksListenAddress.addr} \
             "$@"
         '';
+        defaultText = "(See source)";
       };
 
       # A helper for using doas instead of sudo when doas is enabled
@@ -38,6 +42,7 @@ with lib;
                   # TODO-EXTERNAL: Use absolute path until https://github.com/NixOS/nixpkgs/pull/133622 is available.
                   then "/run/wrappers/bin/doas -u"
                   else "sudo -u";
+        defaultText = "(See source)";
       };
     };
   };

modules/nodeinfo.nix

@@ -8,6 +8,7 @@ let
       program = mkOption {
         readOnly = true;
         default = script;
+        defaultText = "(See source)";
       };
     };
   };

modules/onion-addresses.nix

@@ -28,7 +28,7 @@ let
       default = [];
       description = ''
         Services that can access their onion address via file
-        /var/lib/onion-addresses/<service>
+        `/var/lib/onion-addresses/$service`
         The file is readable only by the service user.
       '';
     };

modules/secrets/secrets.nix

@@ -67,16 +67,7 @@ let
 
     secretsSetupMethod = mkOption {
       type = types.str;
-      default = throw  ''
-        Error: No secrets setup method has been defined.
-        To fix this, choose one of the following:
-
-         - Use one of the deployment methods in ${toString ./../deployment}
-
-         - Set `nix-bitcoin.generateSecrets = true` to automatically generate secrets
-
-         - Set `nix-bitcoin.secretsSetupMethod = "manual"` if you want to manually setup secrets
-      '';
+      default = null;
     };
 
     generateSecretsScript = mkOption {
@@ -141,6 +132,21 @@ in {
   inherit options;
 
   config = {
+    assertions = [
+      { assertion = cfg.secretsSetupMethod != null;
+        message = ''
+          No secrets setup method has been defined.
+          To fix this, choose one of the following:
+
+           - Use one of the deployment methods in ${toString ./../deployment}
+
+           - Set `nix-bitcoin.generateSecrets = true` to automatically generate secrets
+
+           - Set `nix-bitcoin.secretsSetupMethod = "manual"` if you want to manually setup secrets
+        '';
+      }
+    ];
+
     # This target is active when secrets have been setup successfully.
     systemd.targets.nix-bitcoin-secrets = mkIf (cfg.secretsSetupMethod != "manual") {
       # This ensures that the secrets target is always activated when switching

test/ci/build.sh

@@ -7,9 +7,7 @@
 
 set -euo pipefail
 
-scenario=${scenario:-}
-
-if [[ -v CIRRUS_CI && $scenario ]]; then
+if [[ -v CIRRUS_CI ]]; then
     if [[ ! -e /dev/kvm ]]; then
         >&2 echo "No KVM available on VM host."
         exit 1
@@ -20,10 +18,4 @@ fi
 
 echo "$NIX_PATH ($(nix eval --raw nixpkgs.lib.version))"
 
-if [[ $scenario ]]; then
-    testArgs="--scenario $scenario"
-else
-    testArgs=pkgsUnstable
-fi
-
-"${BASH_SOURCE[0]%/*}/../test/run-tests.sh" --ci $testArgs
+"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario $scenario

test/nixos-search/ci-test.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd "${BASH_SOURCE[0]%/*}"
+
+# Use cachix to cache the `flake-info` build
+cachixCache=nix-bitcoin
+
+nix run .#cachix -- use $cachixCache
+
+# We're running in a basic, unprivileged container that doesn't support sandboxing.
+# Sandboxing is unnneeded because we're only building the 3rd-party `flake-info` tool.
+echo "sandbox = false" >> /etc/nix/nix.conf
+export PATH=$(nix shell -L .#flake-info .#cachix -c sh -c 'echo $PATH')
+
+if [[ ${CACHIX_SIGNING_KEY:-} ]]; then
+    cachix push $cachixCache $(type -P flake-info);
+fi
+
+# flake-info requires '<nixpkgs>'
+export NIX_PATH=nixpkgs=$(nix eval --raw .#nixpkgsPath)
+echo "Running flake-info (nixos-search)"
+flake-info flake ./.

test/nixos-search/flake.lock

@@ -0,0 +1,44 @@
+{
+  "nodes": {
+    "nixos-search": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1638831015,
+        "narHash": "sha256-WmCsa6gSY01Yt0wminCbNV8Mn+4iU1OEF5r3YUh6SVs=",
+        "owner": "nixos",
+        "repo": "nixos-search",
+        "rev": "fa5fe2a61c36ed90506af75c19bfaf2f9d537d87",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixos-search",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1631118067,
+        "narHash": "sha256-tEcFvm3a6ToeBGwHdjfB2mVQwa4LZCZTQYE2LnY3ycA=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "09cd65b33c5653d7d2954fef4b9f0e718c899743",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-unstable",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixos-search": "nixos-search"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

test/nixos-search/flake.nix

@@ -0,0 +1,13 @@
+# This flake just mirrors input `nixos-search`.
+# Because `nixos-search` is a dev-only dependency, we don't add
+# it to the main flake.
+{
+  inputs.nixos-search.url = "github:nixos/nixos-search";
+  outputs = { self, nixos-search }: {
+    inherit (nixos-search) packages;
+
+    # Used by ./ci-test.sh
+    inherit (nixos-search.inputs.nixpkgs) legacyPackages;
+    nixpkgsPath = toString nixos-search.inputs.nixpkgs;
+  };
+}

test/run-tests.sh

@@ -199,7 +199,7 @@ doBuild() {
     name=$1
     shift
     if [[ $ciBuild ]]; then
-        "$scriptDir/../ci/build-to-cachix.sh" "$@"
+        "$scriptDir/ci/build-to-cachix.sh" "$@"
     else
         if [[ $outLinkPrefix ]]; then
             outLink="--out-link $outLinkPrefix-$name"
@@ -246,6 +246,39 @@ vmTestNixExpr() {
 EOF
 }
 
+checkFlakeSupport() {
+    testName=$1
+    if [[ ! -v hasFlakes ]]; then
+        if [[ $(nix flake 2>&1) == *"requires a sub-command"* ]]; then
+            hasFlakes=1
+        else
+            hasFlakes=
+        fi
+    fi
+    if [[ ! $hasFlakes ]]; then
+        echo "Skipping test '$testName'. Nix flake support is not enabled."
+        return 1
+    fi
+}
+
+flake() {
+    if ! checkFlakeSupport "flake"; then return; fi
+
+    nix flake check "$scriptDir/.."
+}
+
+# Test generating module documentation for search.nixos.org
+nixosSearch() {
+    if ! checkFlakeSupport "nixosSearch"; then return; fi
+
+    if [[ $outLinkPrefix ]]; then
+        # Add gcroots for flake-info
+        nix build $scriptDir/nixos-search#flake-info -o "$outLinkPrefix-flake-info"
+    fi
+    echo "Running flake-info (nixos-search)"
+    nix run $scriptDir/nixos-search#flake-info -- flake ../.
+}
+
 # A basic subset of tests to keep the total runtime within
 # manageable bounds (<4 min on desktop systems).
 # These are also run on the CI server.
@@ -275,18 +308,11 @@ examples() {
     (cd "$scriptDir/../examples" && nix-shell --run "$script")
 }
 
-flake() {
-    if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then
-        echo "Skipping flake test. Nix flake support is not enabled."
-    else
-        nix flake check "$scriptDir/.."
-    fi
-}
-
 all() {
     buildable
     examples
     flake
+    nixosSearch
 }
 
 # An alias for buildTest