From 85f9886137a0e5e37db993083cb132d10003be5d Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sat, 11 Jun 2022 18:35:47 +0200 Subject: [PATCH] Citadel 0.0.5 (#45) Co-authored-by: Philipp Walter --- .gitignore | 3 -- app/lib/composegenerator/v3/generate.py | 2 +- bin/lncli | 2 +- docker-compose.yml | 7 ++-- events/signals/.gitkeep | 0 .../{use-beta-builds => set-update-channel} | 2 +- events/triggers/use-stable-builds | 9 ---- info.json | 6 +-- scripts/backup/backup | 41 +++++-------------- scripts/backup/decoy-trigger | 2 +- scripts/backup/monitor | 4 +- scripts/memory-usage | 29 +++++++------ scripts/start | 2 +- scripts/status/memory | 7 +++- scripts/update/.updateignore | 1 - services/manage.py | 15 +++++++ 16 files changed, 61 insertions(+), 71 deletions(-) delete mode 100644 events/signals/.gitkeep rename events/triggers/{use-beta-builds => set-update-channel} (79%) delete mode 100755 events/triggers/use-stable-builds diff --git a/.gitignore b/.gitignore index 94b4eb5..5e50f48 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ electrs/* fulcrumx/* nginx/* redis/* -events/signals/* docker-compose.override.yml # Commit these empty directories @@ -36,7 +35,6 @@ docker-compose.override.yml !db/citadel-seed db/citadel-seed/* !db/citadel-seed/.gitkeep -!events/signals/.gitkeep !lnd/.gitkeep !logs/.gitkeep !tor/data/.gitkeep @@ -46,7 +44,6 @@ db/citadel-seed/* !nginx/.gitkeep !redis/.gitkeep !fulcrumx/.gitkeep -!events/signals/.gitkeep !**/*.license services/installed.json diff --git a/app/lib/composegenerator/v3/generate.py b/app/lib/composegenerator/v3/generate.py index 5b7f56e..3e2789f 100644 --- a/app/lib/composegenerator/v3/generate.py +++ b/app/lib/composegenerator/v3/generate.py @@ -90,7 +90,7 @@ def createComposeConfigFromV3(app: dict, nodeRoot: str): del container.requiredPorts for container in newApp.containers: for udpPort in container.requiredUdpPorts: - container.ports.append("{}/udp:{}/udp".format(udpPort, udpPort)) + container.ports.append("{}:{}/udp".format(udpPort, udpPort)) del container.requiredUdpPorts newApp = configureMainPort(newApp, nodeRoot) newApp = configureHiddenServices(newApp, nodeRoot) diff --git a/bin/lncli b/bin/lncli index 3de018f..67753c0 100755 --- a/bin/lncli +++ b/bin/lncli @@ -13,7 +13,7 @@ CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)" result=$(docker compose \ --file "${CITADEL_ROOT}/docker-compose.yml" \ --env-file "${CITADEL_ROOT}/.env" \ - exec lnd lncli "$@") + exec lightning lncli "$@") # We need to echo with quotes to preserve output formatting echo "$result" diff --git a/docker-compose.yml b/docker-compose.yml index ba90ff9..60eb321 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,7 @@ services: ipv4_address: $LND_IP dashboard: container_name: dashboard - image: ghcr.io/runcitadel/dashboard:v0.0.12@sha256:102a44b938765be8224c63785949b90bf796670f01aef4e282d58a4c26a42595 + image: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5 restart: on-failure stop_grace_period: 1m30s networks: @@ -108,7 +108,7 @@ services: ipv4_address: $DASHBOARD_IP manager: container_name: manager - image: ghcr.io/runcitadel/manager:v0.0.13@sha256:3ced2643e12253fea46abb48a69dcd999d69d3d86ed9956ae0298d4f6be4f06d + image: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce depends_on: - tor - redis @@ -120,7 +120,6 @@ services: volumes: - ${PWD}/info.json:/info.json - ${PWD}/db:/db - - ${PWD}/events/signals:/signals - ${PWD}/events:/events - ${PWD}/apps:/apps - ${PWD}/lnd:/lnd:ro @@ -163,7 +162,7 @@ services: ipv4_address: $MANAGER_IP middleware: container_name: middleware - image: ghcr.io/runcitadel/middleware:v0.0.10@sha256:afd6e2b6f5ba27cde32f6f6d630ddc6dd46d1072871f7834d7424d0554d0f53d + image: ghcr.io/runcitadel/middleware:v0.0.11@sha256:e472da8cbfa67d9a9dbf321334fe65cdf20a0f9b6d6bab33fdf07210f54e7002 depends_on: - manager - bitcoin diff --git a/events/signals/.gitkeep b/events/signals/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/events/triggers/use-beta-builds b/events/triggers/set-update-channel similarity index 79% rename from events/triggers/use-beta-builds rename to events/triggers/set-update-channel index 252e9b5..38fdec5 100755 --- a/events/triggers/use-beta-builds +++ b/events/triggers/set-update-channel @@ -6,4 +6,4 @@ CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" -"${CITADEL_ROOT}/scripts/set-update-channel" beta +"${CITADEL_ROOT}/scripts/set-update-channel" "${1}" diff --git a/events/triggers/use-stable-builds b/events/triggers/use-stable-builds deleted file mode 100755 index 9a60e13..0000000 --- a/events/triggers/use-stable-builds +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" - -"${CITADEL_ROOT}/scripts/set-update-channel" stable diff --git a/info.json b/info.json index 5737f16..f35a1c9 100644 --- a/info.json +++ b/info.json @@ -1,6 +1,6 @@ { - "version": "0.0.4", - "name": "Citadel 0.0.4", + "version": "0.0.5", + "name": "Citadel 0.0.5", "requires": ">=0.0.1", - "notes": "This update fixes multiple bugs in the 0.0.3 release." + "notes": "This update fixes a few bugs in the 0.0.4 release that were preventing some apps from working correctly." } diff --git a/scripts/backup/backup b/scripts/backup/backup index a0be32d..035716a 100755 --- a/scripts/backup/backup +++ b/scripts/backup/backup @@ -1,6 +1,7 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: 2020 Umbrel. https://getumbrel.com +# SPDX-FileCopyrightText: 2022 Citadel and contributors. https://runcitadel.space # # SPDX-License-Identifier: GPL-3.0-or-later @@ -98,39 +99,19 @@ tar \ # --verbose \ # --gzip -# Check if a file exists on Firebase by checking if -# "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F?alt=media" -# returns a 200 response code. -# -check_if_exists() { - curl -s -o /dev/null -w "%{http_code}" \ - -X GET \ - "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F${1}?alt=media" -} - -# Upload a file to Firebase Cloud Storage by uloading its name + a random ID. -# Before uploading, we need to check if a file with the same name already exists, and if it does, change the random ID. -# For example, if we want to upload a file named "" with the content AA, we'd use this command to upload: -# curl -X POST "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F?alt=media" -d "AA" -H "Content-Type: text/plain" -# To check if a file exists, we can check if this endpoint returns an error 404: -# curl "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F?alt=media" -# To download a file, we can the same endpoint upload_file() { local file_to_send="${1}" - local file_name="${2}" - # A random ID to avoid collisions - local random_id="$(tr -dc A-Za-z0-9 /dev/null + "https://account.runcitadel.space/api/upload" \ + -d "${upload_data}" \ + -H "Content-Type: application/json" \ + --socks5 "localhost:${TOR_PROXY_PORT}" \ + > /dev/null } if [[ $BITCOIN_NETWORK == "testnet" ]]; then diff --git a/scripts/backup/decoy-trigger b/scripts/backup/decoy-trigger index 9d0465b..ddc8918 100755 --- a/scripts/backup/decoy-trigger +++ b/scripts/backup/decoy-trigger @@ -39,7 +39,7 @@ main () { echo "Sleeping for ${delay} seconds..." sleep $delay echo "Triggering decoy backup..." - touch "${CITADEL_ROOT}/events/signals/backup" + "${CITADEL_ROOT}/scripts/backup/backup" done } diff --git a/scripts/backup/monitor b/scripts/backup/monitor index 5234ac9..6bc499c 100755 --- a/scripts/backup/monitor +++ b/scripts/backup/monitor @@ -48,10 +48,10 @@ monitor_file () { sleep 1 done echo "$file_path created! Triggering backup..." - touch "${CITADEL_ROOT}/events/signals/backup" + "${CITADEL_ROOT}/scripts/backup/backup" fi - fswatch -0 --event Updated $file_path | xargs -0 -n 1 -I {} touch "${CITADEL_ROOT}/events/signals/backup" + fswatch -0 --event Updated $file_path | xargs -0 -n 1 -I {} "${CITADEL_ROOT}/scripts/backup/backup" } if [[ ! -d "${CITADEL_ROOT}" ]]; then diff --git a/scripts/memory-usage b/scripts/memory-usage index 1ff1f49..9c2ea19 100755 --- a/scripts/memory-usage +++ b/scripts/memory-usage @@ -21,7 +21,17 @@ function get_memory_usage() { function mem_usage_to_percent() { local mem_usage="$1" local total_mem="$(free -m | awk 'NR==2 {print $2}')" - echo "$(awk "BEGIN {printf \"%.1f\", $mem_usage / $total_mem * 100}")" + echo "$(awk "BEGIN {printf \"%.1f\", ${mem_usage/,/.} / ${total_mem/,/.} * 100}")" +} + +function app_mem_usage() { + # For every container of the app, get the mem usage, save it, and at the end, print the total mem usage of the app + local mem_usage=0 + for container in $(get_app_containers "$1"); do + # Use awk to add, it supports floating point numbers + mem_usage=$(awk "BEGIN {printf \"%.2f\", $mem_usage + $(get_memory_usage "$container")}") + done + echo "${1}: $mem_usage%" } get_total_used_mem_raw() { @@ -35,7 +45,7 @@ get_total_used_mem() { # To get the containers of the app, list every container whose name starts with the name of the app get_app_containers () { local app_name="$1" - "${CITADEL_ROOT}/scripts/app" compose "${app_name}" ps | awk '{print $1}' | grep -v 'Name\|-----' + "${CITADEL_ROOT}/scripts/app" compose "${app_name}" ps | awk '{print $1}' | grep -v 'NAME' } # Get the memory usage of the whole system, excluding docker containers @@ -48,22 +58,17 @@ get_system_memory_usage() { } main() { - echo "total: $(get_total_used_mem)%"& + echo "total: $(get_total_used_mem)%" + echo "system: $(get_system_memory_usage)%" for service in bitcoin lightning electrum tor; do echo "${service}: $(get_memory_usage "$service")%" & done for app in $("${CITADEL_ROOT}/scripts/app" ls-installed); do - # For every container of the app, get the mem usage, save it, and at the end, print the total mem usage of the app - local mem_usage=0 - for container in $(get_app_containers "$app"); do - # Use awk to add, it supports floating point numbers - mem_usage=$(awk "BEGIN {printf \"%.2f\", $mem_usage + $(get_memory_usage "$container")}") - done - wait - echo "${app}: $mem_usage%" + app_mem_usage "${app}" & done - echo "system: $(get_system_memory_usage)%" wait } +echo "Calculating memory usage..." +echo "This may take a while, please wait..." main | sort --key 2 --numeric-sort --reverse diff --git a/scripts/start b/scripts/start index 63eff7b..983a702 100755 --- a/scripts/start +++ b/scripts/start @@ -90,7 +90,7 @@ echo echo "Starting status monitors..." echo pkill -f ./scripts/status-monitor || true -./scripts/status-monitor memory 60 &>> "${CITADEL_LOGS}/status-monitor.log" & +./scripts/status-monitor memory 300 &>> "${CITADEL_LOGS}/status-monitor.log" & ./scripts/status-monitor storage 60 &>> "${CITADEL_LOGS}/status-monitor.log" & ./scripts/status-monitor temperature 15 &>> "${CITADEL_LOGS}/status-monitor.log" & ./scripts/status-monitor uptime 15 &>> "${CITADEL_LOGS}/status-monitor.log" & diff --git a/scripts/status/memory b/scripts/status/memory index 7ffe607..8b05f15 100755 --- a/scripts/status/memory +++ b/scripts/status/memory @@ -4,6 +4,9 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +# To prevent tools we use from outputting in another language +LANG=C + CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" memory_total_bytes() { @@ -23,7 +26,7 @@ get_app_memory_use() { local app_memory=0 - local app_containers=$("${CITADEL_ROOT}/app/app-manager.py" compose "${app}" ps | awk '{print $1}' | grep -v 'Name\|-----') + local app_containers=$("${CITADEL_ROOT}/scripts/app" compose "${app}" ps | awk '{print $1}' | grep -v 'NAME') for container in $app_containers; do local container_memory=$(get_container_memory_use "${container}") app_memory=$(awk "BEGIN {print ${app_memory}+${container_memory}}") @@ -43,7 +46,7 @@ used=$(memory_used_bytes) json=$(echo $json | jq --arg used "${used}" '. + {used: $used|tonumber}') cumulative_app_memory="0" -for app in $( "${CITADEL_ROOT}/app/app-manager.py" ls-installed); do +for app in $( "${CITADEL_ROOT}/scripts/app" ls-installed); do app_memory=$(get_app_memory_use "${app}") cumulative_app_memory=$(($cumulative_app_memory+$app_memory)) json=$(echo $json | jq --arg app "${app}" --arg app_memory "${app_memory}" '.breakdown |= .+ [{id: $app, used: $app_memory|tonumber}]') diff --git a/scripts/update/.updateignore b/scripts/update/.updateignore index b7915c6..92b78e1 100644 --- a/scripts/update/.updateignore +++ b/scripts/update/.updateignore @@ -5,7 +5,6 @@ lnd/* statuses tor/* electrs/* -events/signals logs/* app-data/* apps/networking.json diff --git a/services/manage.py b/services/manage.py index 3e9a3dc..c965b2d 100755 --- a/services/manage.py +++ b/services/manage.py @@ -31,6 +31,21 @@ args = parser.parse_args() # Function to install a service # To install it, read the service's YAML file (nodeRoot/services/name.yml) and add it to the main compose file (nodeRoot/docker-compose.yml) def setService(name, implementation): + # Get all available services + services = next(os.walk(os.path.join(nodeRoot, "services")))[1] + + if not name in services: + print("\"{}\" is not a valid service.".format(name)) + exit(1) + + # Get all available implementations + implementations = next(os.walk(os.path.join(nodeRoot, "services", name)), (None, None, []))[2] + implementations = [x.split('.')[0] for x in implementations] + + if not implementation in implementations: + print("\"{}\" is not a valid implementation.".format(implementation)) + exit(1) + # Read the YAML file with open(os.path.join(nodeRoot, "services", name, implementation + ".yml"), 'r') as stream: service = yaml.safe_load(stream)