Citadel 0.0.5 (#45)

Co-authored-by: Philipp Walter <philippwalter@pm.me>

.gitignore

@@ -25,7 +25,6 @@ electrs/*
 fulcrumx/*
 nginx/*
 redis/*
-events/signals/*
 docker-compose.override.yml
 
 # Commit these empty directories
@@ -36,7 +35,6 @@ docker-compose.override.yml
 !db/citadel-seed
 db/citadel-seed/*
 !db/citadel-seed/.gitkeep
-!events/signals/.gitkeep
 !lnd/.gitkeep
 !logs/.gitkeep
 !tor/data/.gitkeep
@@ -46,7 +44,6 @@ db/citadel-seed/*
 !nginx/.gitkeep
 !redis/.gitkeep
 !fulcrumx/.gitkeep
-!events/signals/.gitkeep
 
 !**/*.license
 services/installed.json

app/lib/composegenerator/v3/generate.py

@@ -90,7 +90,7 @@ def createComposeConfigFromV3(app: dict, nodeRoot: str):
         del container.requiredPorts
     for container in newApp.containers:
         for udpPort in container.requiredUdpPorts:
-            container.ports.append("{}/udp:{}/udp".format(udpPort, udpPort))
+            container.ports.append("{}:{}/udp".format(udpPort, udpPort))
         del container.requiredUdpPorts
     newApp = configureMainPort(newApp, nodeRoot)
     newApp = configureHiddenServices(newApp, nodeRoot)

bin/lncli

@@ -13,7 +13,7 @@ CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)"
 result=$(docker compose \
   --file "${CITADEL_ROOT}/docker-compose.yml" \
   --env-file "${CITADEL_ROOT}/.env" \
-  exec lnd lncli "$@")
+  exec lightning lncli "$@")
 
 # We need to echo with quotes to preserve output formatting
 echo "$result"

docker-compose.yml

@@ -100,7 +100,7 @@ services:
         ipv4_address: $LND_IP
   dashboard:
     container_name: dashboard
-    image: ghcr.io/runcitadel/dashboard:v0.0.12@sha256:102a44b938765be8224c63785949b90bf796670f01aef4e282d58a4c26a42595
+    image: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5
     restart: on-failure
     stop_grace_period: 1m30s
     networks:
@@ -108,7 +108,7 @@ services:
         ipv4_address: $DASHBOARD_IP
   manager:
     container_name: manager
-    image: ghcr.io/runcitadel/manager:v0.0.13@sha256:3ced2643e12253fea46abb48a69dcd999d69d3d86ed9956ae0298d4f6be4f06d
+    image: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce
     depends_on:
     - tor
     - redis
@@ -120,7 +120,6 @@ services:
     volumes:
     - ${PWD}/info.json:/info.json
     - ${PWD}/db:/db
-    - ${PWD}/events/signals:/signals
     - ${PWD}/events:/events
     - ${PWD}/apps:/apps
     - ${PWD}/lnd:/lnd:ro
@@ -163,7 +162,7 @@ services:
         ipv4_address: $MANAGER_IP
   middleware:
     container_name: middleware
-    image: ghcr.io/runcitadel/middleware:v0.0.10@sha256:afd6e2b6f5ba27cde32f6f6d630ddc6dd46d1072871f7834d7424d0554d0f53d
+    image: ghcr.io/runcitadel/middleware:v0.0.11@sha256:e472da8cbfa67d9a9dbf321334fe65cdf20a0f9b6d6bab33fdf07210f54e7002
     depends_on:
     - manager
     - bitcoin

events/triggers/set-update-channel

@@ -6,4 +6,4 @@
 
 CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
 
-"${CITADEL_ROOT}/scripts/set-update-channel" beta
+"${CITADEL_ROOT}/scripts/set-update-channel" "${1}"

events/triggers/use-stable-builds

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
-
-"${CITADEL_ROOT}/scripts/set-update-channel" stable

info.json

@@ -1,6 +1,6 @@
 {
-    "version": "0.0.4",
+    "version": "0.0.5",
-    "name": "Citadel 0.0.4",
+    "name": "Citadel 0.0.5",
     "requires": ">=0.0.1",
-    "notes": "This update fixes multiple bugs in the 0.0.3 release."
+    "notes": "This update fixes a few bugs in the 0.0.4 release that were preventing some apps from working correctly."
 }

scripts/backup/backup

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # SPDX-FileCopyrightText: 2020 Umbrel. https://getumbrel.com
+# SPDX-FileCopyrightText: 2022 Citadel and contributors. https://runcitadel.space
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -98,39 +99,19 @@ tar \
 #   --verbose \
 #   --gzip
 
-# Check if a file exists on Firebase by checking if
-# "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F<THE_FILE_NAME>?alt=media"
-# returns a 200 response code.
-#
-check_if_exists() {
-  curl -s -o /dev/null -w "%{http_code}" \
-    -X GET \
-    "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F${1}?alt=media"
-}
-
-# Upload a file to Firebase Cloud Storage by uloading its name + a random ID.
-# Before uploading, we need to check if a file with the same name already exists, and if it does, change the random ID.
-# For example, if we want to upload a file named "<THE_FILE_NAME>" with the content AA, we'd use this command to upload:
-# curl -X POST "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F<THE_FILE_NAME>?alt=media" -d "AA" -H "Content-Type: text/plain"
-# To check if a file exists, we can check if this endpoint returns an error 404:
-# curl "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F<THE_FILE_NAME>?alt=media"
-# To download a file, we can the same endpoint
 upload_file() {
   local file_to_send="${1}"
-  local file_name="${2}"
+  local backup_id="${2}"
-  # A random ID to avoid collisions
+  local upload_data=$(jq --null-input \
-  local random_id="$(tr -dc A-Za-z0-9 </dev/urandom | head -c 60 ; echo '')"
+  --arg name "$backup_id" \
-  # Check if the file already exists
+  --arg data "$(base64 $file_to_send)" \
-  # While a file with the same name exists, we'll try to upload it with a different ID
+  '{"name": $name, "data": $data}')
-  while [[ $(check_if_exists "${file_name}-${random_id}") == "200" ]]; do
-    random_id="$(tr -dc A-Za-z0-9 </dev/urandom | head -c 60 ; echo '')"
-  done
-  # Upload the file
   curl -X POST \
-  "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F${file_name}-${random_id}?alt=media" \
+    "https://account.runcitadel.space/api/upload" \
-  -d @"${file_to_send}" \
+    -d "${upload_data}" \
-  -H "Content-Type: application/octet-stream" \
+    -H "Content-Type: application/json" \
-  > /dev/null
+    --socks5 "localhost:${TOR_PROXY_PORT}" \
+    > /dev/null
 }
 
 if [[ $BITCOIN_NETWORK == "testnet" ]]; then

scripts/backup/decoy-trigger

@@ -39,7 +39,7 @@ main () {
     echo "Sleeping for ${delay} seconds..."
     sleep $delay
     echo "Triggering decoy backup..."
-    touch "${CITADEL_ROOT}/events/signals/backup"
+    "${CITADEL_ROOT}/scripts/backup/backup"
   done
 }
 

scripts/backup/monitor

@@ -48,10 +48,10 @@ monitor_file () {
       sleep 1
     done
     echo "$file_path created! Triggering backup..."
-    touch "${CITADEL_ROOT}/events/signals/backup"
+    "${CITADEL_ROOT}/scripts/backup/backup"
   fi
 
-  fswatch -0 --event Updated $file_path | xargs -0 -n 1 -I {} touch "${CITADEL_ROOT}/events/signals/backup"
+  fswatch -0 --event Updated $file_path | xargs -0 -n 1 -I {} "${CITADEL_ROOT}/scripts/backup/backup"
 }
 
 if [[ ! -d "${CITADEL_ROOT}" ]]; then

scripts/memory-usage

@@ -21,7 +21,17 @@ function get_memory_usage() {
 function mem_usage_to_percent() {
   local mem_usage="$1"
   local total_mem="$(free -m | awk 'NR==2 {print $2}')"
-  echo "$(awk "BEGIN {printf \"%.1f\", $mem_usage / $total_mem * 100}")"
+  echo "$(awk "BEGIN {printf \"%.1f\", ${mem_usage/,/.} / ${total_mem/,/.} * 100}")"
+}
+
+function app_mem_usage() {
+  # For every container of the app, get the mem usage, save it, and at the end, print the total mem usage of the app
+    local mem_usage=0
+    for container in $(get_app_containers "$1"); do
+      # Use awk to add, it supports floating point numbers
+      mem_usage=$(awk "BEGIN {printf \"%.2f\", $mem_usage + $(get_memory_usage "$container")}")
+    done
+    echo "${1}: $mem_usage%"
 }
 
 get_total_used_mem_raw() {
@@ -35,7 +45,7 @@ get_total_used_mem() {
 # To get the containers of the app, list every container whose name starts with the name of the app
 get_app_containers () {
   local app_name="$1"
-  "${CITADEL_ROOT}/scripts/app" compose "${app_name}" ps | awk '{print $1}' | grep -v 'Name\|-----'
+  "${CITADEL_ROOT}/scripts/app" compose "${app_name}" ps | awk '{print $1}' | grep -v 'NAME'
 }
 
 # Get the memory usage of the whole system, excluding docker containers
@@ -48,22 +58,17 @@ get_system_memory_usage() {
 }
 
 main() {
-  echo "total: $(get_total_used_mem)%"&
+  echo "total: $(get_total_used_mem)%"
+  echo "system: $(get_system_memory_usage)%"
   for service in bitcoin lightning electrum tor; do
     echo "${service}: $(get_memory_usage "$service")%" &
   done
   for app in $("${CITADEL_ROOT}/scripts/app" ls-installed); do
-    # For every container of the app, get the mem usage, save it, and at the end, print the total mem usage of the app
+    app_mem_usage "${app}" &
-    local mem_usage=0
-    for container in $(get_app_containers "$app"); do
-      # Use awk to add, it supports floating point numbers
-      mem_usage=$(awk "BEGIN {printf \"%.2f\", $mem_usage + $(get_memory_usage "$container")}")
-    done
-    wait
-    echo "${app}: $mem_usage%"
   done
-  echo "system: $(get_system_memory_usage)%"
   wait
 }
 
+echo "Calculating memory usage..."
+echo "This may take a while, please wait..."
 main | sort --key 2 --numeric-sort --reverse

scripts/start

@@ -90,7 +90,7 @@ echo
 echo "Starting status monitors..."
 echo
 pkill -f ./scripts/status-monitor || true
-./scripts/status-monitor memory 60 &>> "${CITADEL_LOGS}/status-monitor.log" &
+./scripts/status-monitor memory 300 &>> "${CITADEL_LOGS}/status-monitor.log" &
 ./scripts/status-monitor storage 60 &>> "${CITADEL_LOGS}/status-monitor.log" &
 ./scripts/status-monitor temperature 15 &>> "${CITADEL_LOGS}/status-monitor.log" &
 ./scripts/status-monitor uptime 15 &>> "${CITADEL_LOGS}/status-monitor.log" &

scripts/status/memory

@@ -4,6 +4,9 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+# To prevent tools we use from outputting in another language
+LANG=C
+
 CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
 
 memory_total_bytes() {
@@ -23,7 +26,7 @@ get_app_memory_use() {
 
   local app_memory=0
 
-  local app_containers=$("${CITADEL_ROOT}/app/app-manager.py" compose "${app}" ps | awk '{print $1}' | grep -v 'Name\|-----')
+  local app_containers=$("${CITADEL_ROOT}/scripts/app" compose "${app}" ps | awk '{print $1}' | grep -v 'NAME')
   for container in $app_containers; do
     local container_memory=$(get_container_memory_use "${container}")
     app_memory=$(awk "BEGIN {print ${app_memory}+${container_memory}}")
@@ -43,7 +46,7 @@ used=$(memory_used_bytes)
 json=$(echo $json | jq --arg used "${used}" '. + {used: $used|tonumber}')
 
 cumulative_app_memory="0"
-for app in $( "${CITADEL_ROOT}/app/app-manager.py" ls-installed); do
+for app in $( "${CITADEL_ROOT}/scripts/app" ls-installed); do
   app_memory=$(get_app_memory_use "${app}")
   cumulative_app_memory=$(($cumulative_app_memory+$app_memory))
   json=$(echo $json | jq --arg app "${app}" --arg app_memory "${app_memory}" '.breakdown |= .+ [{id: $app, used: $app_memory|tonumber}]')

scripts/update/.updateignore

@@ -5,7 +5,6 @@ lnd/*
 statuses
 tor/*
 electrs/*
-events/signals
 logs/*
 app-data/*
 apps/networking.json

services/manage.py

@@ -31,6 +31,21 @@ args = parser.parse_args()
 # Function to install a service
 # To install it, read the service's YAML file (nodeRoot/services/name.yml) and add it to the main compose file (nodeRoot/docker-compose.yml)
 def setService(name, implementation):
+    # Get all available services
+    services = next(os.walk(os.path.join(nodeRoot, "services")))[1]
+
+    if not name in services:
+        print("\"{}\" is not a valid service.".format(name))
+        exit(1)
+
+    # Get all available implementations
+    implementations = next(os.walk(os.path.join(nodeRoot, "services", name)), (None, None, []))[2]
+    implementations = [x.split('.')[0] for x in implementations]
+
+    if not implementation in implementations:
+        print("\"{}\" is not a valid implementation.".format(implementation))
+        exit(1)
+
     # Read the YAML file
     with open(os.path.join(nodeRoot, "services", name, implementation + ".yml"), 'r') as stream:
         service = yaml.safe_load(stream)