tests: optimize building multiple tests at once

The result of `import tests.nix {}` is now an attrset of tests.
This makes it easier and more efficient to evaluate or build multiple
tests in one call to `nix build`.

Simplify tests.nix by removing the large module args scope in favor of
self-contained scenario module definitions.

Add CPU core and memory size defaults to the test configuration to
enable building tests without `run-tests.sh`.

Add the following top-level args to tests.nix:
- `extraScenarios` to provide a nix-level way to define extra scenarios.
- `pkgs` to allow building tests with custom pkgs or systems.

test/lib/make-container.sh

@@ -84,6 +84,6 @@ if [[ ! ($containerBin && $(realpath $containerBin) == *extra-container-0.6*) ]]
 fi
 
 read -d '' src <<EOF || true
-(import "$scriptDir/tests.nix" { scenario = "$scenario"; }).container
+((import "$scriptDir/tests.nix" {}).getTest "$scenario").container
 EOF
 exec extra-container $containerCommand -E "$src" "$@"

test/lib/make-test-vm.nix

@@ -1,7 +1,7 @@
 pkgs:
 let
   pythonTesting = import "${toString pkgs.path}/nixos/lib/testing-python.nix" {
-    system = builtins.currentSystem;
+    system = pkgs.stdenv.hostPlatform.system;
     inherit pkgs;
   };
 in

test/lib/make-test.nix

@@ -1,7 +1,9 @@
+pkgs:
 let
-  pkgs =  import <nixpkgs> { config = {}; overlays = []; };
   makeVM = import ./make-test-vm.nix pkgs;
+  inherit (pkgs) lib;
 in
+
 name: testConfig:
 {
   vm = makeVM {
@@ -9,8 +11,15 @@ name: testConfig:
 
     machine = {
       imports = [ testConfig ];
+      virtualisation = {
         # Needed because duplicity requires 270 MB of free temp space, regardless of backup size
-      virtualisation.diskSize = 1024;
+        diskSize = 1024;
+
+        # Min. 800 MiB needed to avoid 'out of memory' errors
+        memorySize = lib.mkDefault 2048;
+
+        cores = lib.mkDefault 2;
+      };
     };
 
     testScript = nodes: let

test/run-tests.sh

@@ -110,7 +110,7 @@ trap 'eval "$runAtExit"' EXIT
 if [[ $scenario = *' '* ]]; then
     export scenarioOverridesFile=$(mktemp ${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX)
     runAtExit+='rm -f "$scenarioOverridesFile";'
-    echo "{ testEnv, config, pkgs, lib }: with testEnv; with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
+    echo "{ scenarios, pkgs, lib }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
     scenario=tmp
 fi
 
@@ -120,7 +120,7 @@ run() {
     export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
     runAtExit+="rm -rf $TMPDIR;"
 
-    nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver
+    nix-build --out-link $TMPDIR/driver -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver
 
     # Variable 'tests' contains the Python code that is executed by the driver on startup
     if [[ $1 == --interactive ]]; then
@@ -212,7 +212,7 @@ vmTestNixExpr() {
     fi
 
     cat <<EOF
-    (import "$scriptDir/tests.nix" { scenario = "$scenario"; }).vm.overrideAttrs (old: rec {
+    ((import "$scriptDir/tests.nix" {}).getTest "$scenario").vm.overrideAttrs (old: rec {
       buildCommand = ''
         export QEMU_OPTS="-smp $numCPUs -m $memoryMiB $extraQEMUOpts"
         echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"

test/tests.nix

@@ -1,14 +1,20 @@
 # Integration tests, can be run without internet access.
 
-{ scenario ? "default" }:
+let
+  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
+in
 
-import ./lib/make-test.nix scenario (
+{ extraScenarios ? { ... }: {}
-{ config, pkgs, lib, ... }: with lib;
+, pkgs ? import nixpkgs { config = {}; overlays = []; }
-let testEnv = rec {
+}:
+with pkgs.lib;
+let
+  globalPkgs = pkgs;
+
+  baseConfig = { pkgs, config, ... }: let
     cfg = config.services;
     mkIfTest = test: mkIf (config.tests.${test} or false);
-
+  in {
-  baseConfig = {
     imports = [
       ./lib/test-lib.nix
       ../modules/modules.nix
@@ -26,6 +32,9 @@ let testEnv = rec {
     };
 
     config = mkMerge [{
+      # Share the same pkgs instance among tests
+      nixpkgs.pkgs = mkDefault globalPkgs;
+
       tests.bitcoind = cfg.bitcoind.enable;
       services.bitcoind = {
         enable = true;
@@ -183,14 +192,14 @@ let testEnv = rec {
       ];
     };
 
-    netnsBase = {
+    netnsBase = { config, pkgs, ... }: {
       nix-bitcoin.netns-isolation.enable = true;
       test.data.netns = config.nix-bitcoin.netns-isolation.netns;
       tests.netns-isolation = true;
       environment.systemPackages = [ pkgs.fping ];
     };
 
-    regtestBase = {
+    regtestBase = { config, ... }: {
       tests.regtest = true;
 
       services.bitcoind.regtest = true;
@@ -241,20 +250,29 @@ let testEnv = rec {
       # You can also set the env var `scenarioOverridesFile` (used below) to define custom scenarios.
     };
   };
-};
+
-in
-  let
   overrides = builtins.getEnv "scenarioOverridesFile";
-    scenarios = testEnv.scenarios // (optionalAttrs (overrides != "") (import overrides {
+  extraScenarios' = (if (overrides != "") then import overrides else extraScenarios) {
-      inherit testEnv config pkgs lib;
+    inherit scenarios pkgs;
-    }));
+    inherit (pkgs) lib;
-    autoScenario = {
-      services.${scenario}.enable = true;
   };
-  in {
+  allScenarios = scenarios // extraScenarios';
+
+  makeTest = name: config:
+    makeTest' name {
       imports = [
-      scenarios.base
+        allScenarios.base
-      (scenarios.${scenario} or autoScenario)
+        config
       ];
+    };
+  makeTest' = import ./lib/make-test.nix pkgs;
+
+  tests = builtins.mapAttrs makeTest allScenarios;
+
+  getTest = name: tests.${name} or (makeTest name {
+    services.${name}.enable = true;
+  });
+in
+  tests // {
+    inherit getTest;
   }
-)