add modules test

The build of package 'nodeinfo' is implicitly tested by the modules test.
2020-01-12 20:52:40 +01:00 · 2020-01-12 20:52:40 +01:00 · 187ff884db
commit 187ff884db
parent 826245484e
6 changed files with 364 additions and 2 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,3 +1,4 @@
+dist: bionic # needed for KVM
 language: shell

 install:
@ -18,7 +19,7 @@ env:
    # CACHIX_SIGNING_KEY
  - secure: "xXCFZ7g+k5YmCGm8R8l3bZElVmt+RD1KscG3kGr5w4HyyDPTzFetPo+sT8bUpysDU0u3HWhfVhHtpog2mhNhwVl3tQwKXea3dHKC1i6ypBg3gjDngmJRR5wo++ocYDpK8qPaU7m/jHQTNFnTA4CbmMcc05GcYx/1Ai/ZGkNwWFjdIcVeOUoiol33gykMOXIGDg2qlXudt33wP53FHbX8L4fxzodWfAuxKK4AoGprxy5eSnU7LCaXxxJmu4HwuV+Ux2U1NfE/E33cvhlUvTQCswVSZFG06mg8rwhMG1ozsDvlL2itZlu/BeUQH5y3XMMlnJIUXUazkRBibf1w/ebVjpOF+anqkqmq8tcbFEa7T+RJeVTIsvP+L8rE8fcmuZtdg9hNmgRnLmaeT0vVwD1L2UqW9HdRyujdoS0jPYuoc1W7f1JQWfAPhBPQ1SrtKyNNqcbVJ34aN7b+4vCzRpQL1JTbmjzQIWhkiKN1qMo1v/wbIydW8yka4hc4JOfdQLaAJEPI1eAC1MLotSAegMnwKWE1dzm66MuPSipksYjZrvsB28cV4aCVUffIuRhrSr1i2afRHwTpNbK9U4/576hah15ftUdR79Sfkcoi1ekSQTFGRvkRIPYtkKLYwFa3jVA41qz7+IIZCf4TsApy3XDdFx91cRub7yPq9BeZ83A+qYQ="
  jobs:
-  - PKG=nodeinfo STABLE=1
+  - TestModules=1 STABLE=1
  - PKG=hwi STABLE=1
  - PKG=lightning-charge STABLE=1
  - PKG=lightning-charge STABLE=0
@ -33,7 +34,21 @@ env:
  - PKG=liquid-swap STABLE=1
 script:
  - printf '%s (%s)\n' "$NIX_PATH" "$VER"
-  - time nix-instantiate -A $PKG --add-root ./drv --indirect
+  - |
+    getBuildExpr() {
+        if [[ $TestModules ]]; then
+            if [[ ! -e /dev/kvm ]]; then
+                >&2 echo "No KVM available on VM Host."
+                exit 1
+            fi
+            sudo chmod go+rw /dev/kvm
+            test/run-tests.sh exprForCI
+        else
+            echo "(import ./. {}).$PKG"
+        fi
+    }
+  - buildExpr=$(getBuildExpr)
+  - time nix-instantiate -E "$buildExpr" --add-root ./drv --indirect
  - outPath=$(nix-store --query ./drv)
  - |
    if nix path-info --store https://nix-bitcoin.cachix.org $outPath &>/dev/null; then
--- a/test/make-test.nix
+++ b/test/make-test.nix
@ -0,0 +1,44 @@
+testArgs:
+
+let
+  pkgs = import <nixpkgs> { config = {}; overlays = []; };
+
+  # Stable nixpkgs doesn't yet include the Python testing framework.
+  # Use unstable nixpkgs and patch it so that it uses stable nixpkgs for the VM
+  # machine configuration.
+  testingPkgs = let
+    # unstable as of 2020-01-09
+    rev = "9beb0d1ac2ebd6063efbdc4d3631f8ce137bbf90";
+    src = builtins.fetchTarball {
+        url = "https://github.com/nixos/nixpkgs-channels/archive/${rev}.tar.gz";
+        sha256 = "1v95779di35qhrz70p2v27kmwm09h8pgh74i1wc72v0zp3bdrf50";
+    };
+  in
+    pkgs.runCommand "nixpkgs-testing" {} ''
+    cp -r ${src} $out
+    cd $out
+    chmod +w -R .
+    patch -p1 < ${./use-stable-pkgs.patch}
+  '';
+
+  test = (import "${testingPkgs}/nixos/tests/make-test-python.nix") testArgs;
+
+  # Fix the black Python code formatter that's used in the test to allow the test
+  # script to have longer lines. The default width of 88 chars is too restrictive for
+  # our script.
+  fixedTest = { system ? builtins.currentSystem, ... }@args:
+    let
+       pkgs = (import testingPkgs { inherit system; config = {}; overlays = []; } );
+       pkgsFixed = pkgs // {
+         python3Packages = pkgs.python3Packages // {
+           black = pkgs.writeScriptBin "black" ''
+             fileToCheck=''${@:$#}
+             [[ $fileToCheck = *test-script ]] && extraArgs='--line-length 100'
+             exec ${pkgs.python3Packages.black}/bin/black $extraArgs "$@"
+           '';
+         };
+       };
+    in
+      test (args // { pkgs = pkgsFixed; });
+in
+  fixedTest
--- a/test/run-tests.sh
+++ b/test/run-tests.sh
@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+# Modules integration test runner.
+# The test (./test.nix) uses the NixOS testing framework and is executed in a VM.
+#
+# Usage:
+#   Run test
+#   ./run-tests.sh
+#
+#   Run test and save result to avoid garbage collection
+#   ./run-tests.sh build --out-link /tmp/nix-bitcoin-test
+#
+#   Run interactive test debugging
+#   ./run-tests.sh debug
+#
+#   This starts the testing VM and drops you into a Python REPL where you can
+#   manually execute the tests from ./test-script.py
+
+set -eo pipefail
+
+numCPUs=${numCPUs:-$(nproc)}
+# Min. 800 MiB needed to avoid 'out of memory' errors
+memoryMiB=${memoryMiB:-2048}
+
+scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
+
+getPkgs() {
+    nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" $1
+}
+export NIX_PATH=nixpkgs=$(getPkgs nixpkgs):nixpkgs-unstable=$(getPkgs nixpkgs-unstable)
+
+# Run the test. No temporary files are left on the host system.
+run() {
+    # TMPDIR is also used by the test driver for VM tmp files
+    export TMPDIR=$(mktemp -d -p /tmp nix-bitcoin-test.XXXXXX)
+    trap "rm -rf $TMPDIR" EXIT
+
+    nix-build --out-link $TMPDIR/driver "$scriptDir/test.nix" -A driver
+
+    # Variable 'tests' contains the Python code that is executed by the driver on startup
+    if [[ $1 == --interactive ]]; then
+        echo "Running interactive testing environment"
+        tests=$(
+            echo 'is_interactive = True'
+            # The test script raises an error when 'is_interactive' is defined so
+            # that it just loads the initial helper functions and stops before
+            # executing the actual tests
+            echo 'try:'
+            echo '    exec(os.environ["testScript"])'
+            echo 'except:'
+            echo '    pass'
+            # Start VM
+            echo 'start_all()'
+            # Start REPL
+            echo 'import code'
+            echo 'code.interact(local=globals())'
+        )
+    else
+        tests='exec(os.environ["testScript"])'
+    fi
+
+    echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
+    [[ $NB_TEST_ENABLE_NETWORK ]] || QEMU_NET_OPTS='restrict=on'
+    env -i \
+        NIX_PATH="$NIX_PATH" \
+        TMPDIR="$TMPDIR" \
+        tests="$tests" \
+        QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS"  \
+        QEMU_NET_OPTS="$QEMU_NET_OPTS" \
+        $TMPDIR/driver/bin/nixos-test-driver
+}
+
+debug() {
+    run --interactive
+}
+
+# Run the test by building the test derivation
+build() {
+    vmTestNixExpr | nix-build --no-out-link "$@" -
+}
+
+# On continuous integration nodes there are few other processes running alongside the
+# test, so use more memory here for maximum performance.
+exprForCI() {
+    memoryMiB=3072
+    memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo)
+    memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
+    # Round down to nearest multiple of 50 MiB for improved test build caching
+    ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50))
+    ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB
+    >&2 echo "Host memory: total $((memTotalKiB / 1024)) MiB, available $memAvailableMiB MiB, VM $memoryMiB MiB"
+    vmTestNixExpr
+}
+
+vmTestNixExpr() {
+  cat <<EOF
+    (import "$scriptDir/test.nix" {}).overrideAttrs (old: rec {
+      buildCommand = ''
+        export QEMU_OPTS="-smp $numCPUs -m $memoryMiB"
+        echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
+      '' + old.buildCommand;
+    })
+EOF
+}
+
+eval "${@:-build}"
--- a/test/test-script.py
+++ b/test/test-script.py
@ -0,0 +1,104 @@
+def succeed(*cmds):
+    """Returns the concatenated output of all cmds"""
+    return machine.succeed(*cmds)
+
+
+def assert_matches(cmd, regexp):
+    out = succeed(cmd)
+    if not re.search(regexp, out):
+        raise Exception(f"Pattern '{regexp}' not found in '{out}'")
+
+
+def log_has_string(unit, str):
+    return f"journalctl -b --output=cat -u {unit} --grep='{str}'"
+
+
+def assert_no_failure(unit):
+    """Unit should not have failed since the system is running"""
+    machine.fail(log_has_string(unit, "Failed with result"))
+
+
+def assert_running(unit):
+    machine.wait_for_unit(unit)
+    assert_no_failure(unit)
+
+
+# Don't execute the following test suite when this script is running in interactive mode
+if "is_interactive" in vars():
+    raise Exception()
+
+### Tests
+
+assert_running("setup-secrets")
+# Unused secrets should be inaccessible
+succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]')
+
+assert_running("bitcoind")
+machine.wait_until_succeeds("bitcoin-cli getnetworkinfo")
+assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')
+
+assert_running("electrs")
+machine.wait_for_open_port(4224)  # prometeus metrics provider
+assert_running("nginx")
+# SSL stratum server via nginx. Only check for open port, no content is served here
+# as electrs isn't ready.
+machine.wait_for_open_port(50003)
+# Stop electrs from spamming the test log with 'wait for bitcoind sync' messages
+succeed("systemctl stop electrs")
+
+assert_running("liquidd")
+machine.wait_until_succeeds("elements-cli getnetworkinfo")
+assert_matches("su operator -c 'elements-cli getnetworkinfo' | jq", '"version"')
+succeed("su operator -c 'liquidswap-cli --help'")
+
+assert_running("clightning")
+assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
+
+assert_running("spark-wallet")
+spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
+machine.wait_for_open_port(9737)
+assert_matches(f"curl -s {spark_auth}@localhost:9737", "Spark")
+
+assert_running("lightning-charge")
+charge_auth = re.search("API_TOKEN=(.*)", succeed("cat /secrets/lightning-charge-env"))[1]
+machine.wait_for_open_port(9112)
+assert_matches(f"curl -s api-token:{charge_auth}@localhost:9112/info | jq", '"id"')
+
+assert_running("nanopos")
+machine.wait_for_open_port(9116)
+assert_matches("curl localhost:9116", "tshirt")
+
+assert_running("onion-chef")
+
+# FIXME: use 'wait_for_unit' because 'create-web-index' always fails during startup due
+# to incomplete unit dependencies.
+# 'create-web-index' implicitly tests 'nodeinfo'.
+machine.wait_for_unit("create-web-index")
+machine.wait_for_open_port(80)
+assert_matches("curl localhost", "nix-bitcoin")
+assert_matches("curl -L localhost/store", "tshirt")
+
+machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist"))
+assert_no_failure("bitcoind-import-banlist")
+
+### Additional tests
+
+# Current time in µs
+pre_restart = succeed("date +%s.%6N").rstrip()
+
+# Sanity-check system by restarting all services
+succeed("systemctl restart bitcoind clightning spark-wallet lightning-charge nanopos liquidd")
+
+# Now that the bitcoind restart triggered a banlist import restart, check that
+# re-importing already banned addresses works
+machine.wait_until_succeeds(
+    log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist")
+)
+assert_no_failure("bitcoind-import-banlist")
+
+### Test lnd
+
+succeed("systemctl stop nanopos lightning-charge spark-wallet clightning")
+succeed("systemctl start lnd")
+assert_matches("su operator -c 'lncli getinfo' | jq", '"version"')
+assert_no_failure("lnd")
--- a/test/test.nix
+++ b/test/test.nix
@ -0,0 +1,52 @@
+# Integration test, can be run without internet access.
+
+import ./make-test.nix rec {
+  name = "nix-bitcoin";
+
+  hardened = {
+    imports = [ <nixpkgs/nixos/modules/profiles/hardened.nix> ];
+    security.allowUserNamespaces = true; # reenable disabled option
+  };
+
+  machine = { pkgs, lib, ... }: with lib; {
+    imports = [
+      ../modules/nix-bitcoin.nix
+      ../modules/secrets/generate-secrets.nix
+      # using the hardened profile increases total test duration by ~50%, so disable it for now
+      # hardened
+    ];
+
+    services.nix-bitcoin.enable = true;
+    services.bitcoind.extraConfig = mkForce "connect=0";
+
+    services.clightning.enable = true;
+    services.spark-wallet.enable = true;
+    services.lightning-charge.enable = true;
+    services.nanopos.enable = true;
+
+    services.lnd.enable = true;
+    systemd.services.lnd.wantedBy = mkForce [];
+
+    services.electrs.enable = true;
+
+    services.liquidd = {
+      enable = true;
+      listen = mkForce false;
+      extraConfig = "noconnect=1";
+    };
+
+    services.nix-bitcoin-webindex.enable = true;
+
+    services.hardware-wallets = {
+      trezor = true;
+      ledger = true;
+    };
+
+    # to test that unused secrets are made inaccessible by 'setup-secrets'
+    systemd.services.generate-secrets.postStart = ''
+      install -o nobody -g nogroup -m777 <(:) /secrets/dummy
+    '';
+  };
+
+  testScript = builtins.readFile ./test-script.py;
+}
--- a/test/use-stable-pkgs.patch
+++ b/test/use-stable-pkgs.patch
@ -0,0 +1,41 @@
+--- a/nixos/lib/build-vms.nix
+++ b/nixos/lib/build-vms.nix
+@@ -30,10 +30,10 @@ rec {
+   buildVM =
+     nodes: configurations:
+ 
+-    import ./eval-config.nix {
+    import <nixpkgs/nixos/lib/eval-config.nix> {
+       inherit system;
+       modules = configurations ++ extraConfigurations;
+-      baseModules =  (import ../modules/module-list.nix) ++
+      baseModules = (import <nixpkgs/nixos/modules/module-list.nix>) ++
+         [ ../modules/virtualisation/qemu-vm.nix
+           ../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
+           { key = "no-manual"; documentation.nixos.enable = false; }
+
+
+services.connman doesn't exist in stable nixpkgs
+--- a/nixos/modules/virtualisation/qemu-vm.nix
+++ b/nixos/modules/virtualisation/qemu-vm.nix
+@@ -620,7 +620,6 @@ in
+ 
+     # Wireless won't work in the VM.
+     networking.wireless.enable = mkVMOverride false;
+-    services.connman.enable = mkVMOverride false;
+ 
+     # Speed up booting by not waiting for ARP.
+     networking.dhcpcd.extraConfig = "noarp";
+
+The test driver assumed coreutils to be in PATH. This fix will be ported to upstream.
+--- a/nixos/lib/testing-python.nix
+++ b/nixos/lib/testing-python.nix
+@@ -155,7 +155,7 @@ in rec {
+             --add-flags "''${vms[*]}" \
+             ${lib.optionalString enableOCR
+               "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \
+-            --run "export testScript=\"\$(cat $out/test-script)\"" \
+            --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \
+             --set VLANS '${toString vlans}'
+           ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
+           wrapProgram $out/bin/nixos-run-vms \