Citadel 0.0.7 (#76)

Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com>
Co-authored-by: Aaron Dewes <aaron.dewes@protonmail.com>
Co-authored-by: Philipp Walter <philippwalter@pm.me>

.gitignore

@@ -22,7 +22,7 @@ statuses/*
 app-data
 apps
 electrs/*
-fulcrumx/*
+fulcrum/*
 nginx/*
 redis/*
 docker-compose.override.yml
@@ -43,7 +43,7 @@ db/citadel-seed/*
 !db/.gitkeep
 !nginx/.gitkeep
 !redis/.gitkeep
-!fulcrumx/.gitkeep
+!fulcrum/.gitkeep
 
 !**/*.license
 services/installed.json

app/lib/manage.py

@@ -118,26 +118,6 @@ def getUserData():
             userData = json.load(f)
     return userData
 
-def checkUpdateAvailable(name: str) -> bool:
-    latestAppYml = yaml.safe_load(getAppYml(name))
-    with open(os.path.join(appsDir, name, "app.yml"), "r") as f:
-        originalAppYml = yaml.safe_load(f)
-    if not "metadata" in latestAppYml or not "version" in latestAppYml["metadata"] or not "metadata" in originalAppYml or not "version" in originalAppYml["metadata"]:
-        print("App {} is not valid".format(name))
-        return False
-    return semver.compare(latestAppYml["metadata"]["version"], originalAppYml["metadata"]["version"]) > 0
-
-def getAvailableUpdates():
-    availableUpdates = []
-    apps = findAndValidateApps(appsDir)
-    for app in apps:
-        try:
-            if checkUpdateAvailable(app):
-                availableUpdates.append(app)
-        except Exception:
-            print("Warning: Can't check app {} yet".format(app), file=sys.stderr)
-    return availableUpdates
-
 def startInstalled():
     # If userfile doesn't exist, just do nothing
     userData = {}
@@ -364,3 +344,59 @@ def updateRepos():
         shutil.rmtree(tempDir)
     with open(os.path.join(appsDir, "sourceMap.json"), "w") as f:
         json.dump(sourceMap, f)
+
+
+def getAvailableUpdates():
+    availableUpdates = {}
+    repos = []
+    ignoreApps = []
+    with open(sourcesList) as f:
+        repos = f.readlines()
+    try:
+        with open(updateIgnore) as f:
+            ignoreApps = f.readlines()
+    except: pass
+    # For each repo, clone the repo to a temporary dir, checkout the branch,
+    # and overwrite the current app dir with the contents of the temporary dir/apps/app
+    # Set this to ignoreApps. Normally, it keeps track of apps already installed from repos higher in the list,
+    # but apps specified in updateignore have the highest priority
+    alreadyDefined = [s.strip() for s in ignoreApps]
+    for repo in repos:
+        repo = repo.strip()
+        if repo == "":
+            continue
+        # Also ignore comments
+        if repo.startswith("#"):
+            continue
+        # Split the repo into the git url and the branch
+        repo = repo.split(" ")
+        if len(repo) != 2:
+            print("Error: Invalid repo format in " + sourcesList, file=sys.stderr)
+            exit(1)
+        gitUrl = repo[0]
+        branch = repo[1]
+        # Clone the repo to a temporary dir
+        tempDir = tempfile.mkdtemp()
+        # Git clone with a depth of 1 to avoid cloning the entire repo
+        # Don't print anything to stdout, as we don't want to see the git clone output
+        subprocess.run("git clone --depth 1 --branch {} {} {}".format(branch, gitUrl, tempDir), shell=True, stdout=subprocess.DEVNULL)
+        # Overwrite the current app dir with the contents of the temporary dir/apps/app
+        for app in os.listdir(os.path.join(tempDir, "apps")):
+            try:
+                # if the app is already installed (or a simple file instead of a valid app), skip it
+                if app in alreadyDefined or not os.path.isdir(os.path.join(tempDir, "apps", app)):
+                    continue
+                with open(os.path.join(appsDir, app, "app.yml"), "r") as f:
+                    originalAppYml = yaml.safe_load(f)
+                with open(os.path.join(tempDir, "apps", app, "app.yml"), "r") as f:
+                    latestAppYml = yaml.safe_load(f)
+                if semver.compare(latestAppYml["metadata"]["version"], originalAppYml["metadata"]["version"]) > 0:
+                    availableUpdates[app] = {
+                        "updateFrom": originalAppYml["metadata"]["version"],
+                        "updateTo": latestAppYml["metadata"]["version"]
+                    }
+            except Exception:
+                print("Warning: Can't check app {} (yet)".format(app), file=sys.stderr)
+        # Remove the temporary dir
+        shutil.rmtree(tempDir)
+    return availableUpdates

cli/citadel

@@ -375,12 +375,12 @@ if [[ "$command" = "configure" ]]; then
     exit
   fi
 
-  if [[ "$2" = "fulcrumx" ]]; then
+  if [[ "$2" = "fulcrum" ]]; then
     if $persist; then
-      edit_file $CITADEL_ROOT/templates/fulcrumx-sample.conf
+      edit_file $CITADEL_ROOT/templates/fulcrum-sample.conf
       prompt_apply_config electrum true
     else
-      edit_file $CITADEL_ROOT/fulcrumx/fulcrumx.conf
+      edit_file $CITADEL_ROOT/fulcrum/fulcrum.conf
       prompt_apply_config electrum false
     fi
     exit

docker-compose.yml

@@ -65,7 +65,7 @@ services:
         ipv4_address: $NGINX_IP
   bitcoin:
     container_name: bitcoin
-    image: nolim1t/bitcoinknots:v22.0.knots20211108@sha256:a475da2b2ecda55fcc65ea23e1a36c58b2c10549f1c3d3bb3c31c7dda1127354
+    image: ghcr.io/runcitadel/bitcoinknots:main@sha256:5fbee0f6f0d09d42aacc11c373ffe6162210c42ce21e6eba294e547e3ad80219
     depends_on:
     - tor
     volumes:
@@ -79,7 +79,7 @@ services:
         ipv4_address: $BITCOIN_IP
   lightning:
     container_name: lightning
-    image: lightninglabs/lnd:v0.14.3-beta@sha256:6a2234b0aad4caed3d993736816b198d6228f32c59b27ba2218d5ebf516ae905
+    image: lightninglabs/lnd:v0.15.0-beta@sha256:d227a9db0727ff56020c8d6604c8c369757123d238ab6ce679579c2dd0d0d259
     user: 1000:1000
     depends_on:
     - tor

info.json

@@ -1,6 +1,6 @@
 {
-    "version": "0.0.6",
-    "name": "Citadel 0.0.6",
+    "version": "0.0.7",
+    "name": "Citadel 0.0.7",
     "requires": ">=0.0.1",
-    "notes": "This update fixes a security issue in Tor which could lead to slower Tor performance or your node being inaccessible via Tor."
+    "notes": "While we are busy with the next huge update, you may need to wait longer for updates. This update updates Bitcoin Knots and LND to their latest versions to ensure apps can utilize their latest features. In addition, this update includes the Citadel CLI. More information on that will be published soon."
 }

scripts/configure

@@ -334,7 +334,7 @@ templates_to_build = {
   "./templates/bitcoin-sample.conf": "./bitcoin/bitcoin.conf",
   "./templates/.env-sample": "./.env",
   "./templates/electrs-sample.toml": "./electrs/electrs.toml",
-  "./templates/fulcrumx-sample.conf": "./fulcrumx/fulcrumx.conf",
+  "./templates/fulcrum-sample.conf": "./fulcrum/fulcrum.conf",
   "./templates/nginx-sample.conf": "./nginx/nginx.conf"
 }
 

scripts/start

@@ -94,7 +94,7 @@ pkill -f ./scripts/status-monitor || true
 ./scripts/status-monitor storage 60 &>> "${CITADEL_LOGS}/status-monitor.log" &
 ./scripts/status-monitor temperature 15 &>> "${CITADEL_LOGS}/status-monitor.log" &
 ./scripts/status-monitor uptime 15 &>> "${CITADEL_LOGS}/status-monitor.log" &
-./scripts/status-monitor app-updates 300 &>> "${CITADEL_LOGS}/status-monitor.log" &
+./scripts/status-monitor app-updates 600 &>> "${CITADEL_LOGS}/status-monitor.log" &
 
 echo "Starting backup monitor..."
 echo

scripts/update/.updateignore

@@ -12,4 +12,4 @@ nginx/*
 services/installed.yml
 apps/sourceMap.json
 apps/.updateignore
-fulcrumx/*
+fulcrum/*

services/bitcoin/citadel.yml

@@ -1,18 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-bitcoin:
-  container_name: bitcoin
-  image: ghcr.io/runcitadel/bitcoin-custom:main@sha256:d0af506f8dc92a434e845305ac4252d0601b699c4b3bc4443073a0a2e237f3a0
-  depends_on:
-  - tor
-  volumes:
-  - ${PWD}/bitcoin:/data/.bitcoin
-  restart: on-failure
-  stop_grace_period: 1m
-  ports:
-  - $BITCOIN_P2P_PORT:$BITCOIN_P2P_PORT
-  networks:
-    default:
-      ipv4_address: $BITCOIN_IP

services/bitcoin/core.yml

@@ -4,7 +4,7 @@
 
 bitcoin:
   container_name: bitcoin
-  image: lncm/bitcoind:v22.0@sha256:37a1adb29b3abc9f972f0d981f45e41e5fca2e22816a023faa9fdc0084aa4507
+  image: lncm/bitcoind:v23.0@sha256:57317c90d89156a30327fe1b8e51b836e0fd1a8ba13721eb2e75e6b35a570e26
   depends_on:
   - tor
   volumes:

services/bitcoin/knots.yml

@@ -4,7 +4,7 @@
 
 bitcoin:
   container_name: bitcoin
-  image: nolim1t/bitcoinknots:v22.0.knots20211108@sha256:a475da2b2ecda55fcc65ea23e1a36c58b2c10549f1c3d3bb3c31c7dda1127354
+  image: ghcr.io/runcitadel/bitcoinknots:main@sha256:5fbee0f6f0d09d42aacc11c373ffe6162210c42ce21e6eba294e547e3ad80219
   depends_on:
   - tor
   volumes:

services/electrum/fulcrum.yml

@@ -4,12 +4,12 @@
 
 electrum:
   container_name: electrum
-  image: ghcr.io/runcitadel/fulcrumx:latest@sha256:a74abdfe8397f02482faed6bd828477c452df071129f66ad6596d0ab8d29cf39
+  image: cculianu/fulcrum:latest@sha256:c0543f8b8a5bf6b0c447d8525d6b4360a6c07532f7741f19cc2c179968e71848
   working_dir: /data
   volumes:
     - ${PWD}/bitcoin:/bitcoin:ro
-    - ${PWD}/fulcrumx:/data
-  command: /usr/bin/FulcrumX /data/fulcrumx.conf
+    - ${PWD}/fulcrum:/data
+  command: /usr/bin/Fulcrum /data/fulcrum.conf
   restart: on-failure
   stop_grace_period: 5m
   ports:

services/electrum/fulcrumx.yml

@@ -1,19 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-electrum:
-  container_name: electrum
-  image: ghcr.io/runcitadel/fulcrumx:latest@sha256:557a54a5652b475b01c59eb6cffee3568f18b74d875c4cc125e5ac190c4b0706
-  working_dir: /data
-  volumes:
-    - ${PWD}/bitcoin:/bitcoin:ro
-    - ${PWD}/fulcrumx:/data
-  command: /usr/bin/FulcrumX /data/fulcrumx.conf
-  restart: on-failure
-  stop_grace_period: 5m
-  ports:
-      - "$ELECTRUM_PORT:$ELECTRUM_PORT"
-  networks:
-      default:
-          ipv4_address: $ELECTRUM_IP

services/lightning/lnd.yml

@@ -1,6 +1,6 @@
 lightning:
   container_name: lightning
-  image: lightninglabs/lnd:v0.14.3-beta@sha256:6a2234b0aad4caed3d993736816b198d6228f32c59b27ba2218d5ebf516ae905
+  image: lightninglabs/lnd:v0.15.0-beta@sha256:d227a9db0727ff56020c8d6604c8c369757123d238ab6ce679579c2dd0d0d259
   user: 1000:1000
   depends_on:
   - tor