From 53545ba21a380ca383b57dac0b7dc371fe65927f Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Wed, 15 Jun 2022 18:05:56 +0200 Subject: [PATCH] Merge stable to the disable-healthcheck branch (#55) Co-authored-by: Philipp Walter Co-authored-by: nolim1t - f6287b82CC84bcbd --- .gitignore | 4 +- app/lib/citadelutils.py | 8 +- app/lib/composegenerator/next/stage1.py | 24 +++++ app/lib/composegenerator/shared/env.py | 19 ++-- .../composegenerator/v2/utils/networking.py | 8 +- app/lib/composegenerator/v3/generate.py | 32 +++--- app/lib/composegenerator/v3/networking.py | 2 + app/lib/composegenerator/v3/types.py | 2 +- app/lib/metadata.py | 4 + app/lib/validate.py | 16 +-- bin/lncli | 2 +- docker-compose.yml | 16 +-- events/signals/.gitkeep | 0 events/triggers/app | 6 +- .../{use-beta-builds => set-update-channel} | 2 +- events/triggers/use-stable-builds | 9 -- info.json | 6 +- karen | 100 ++++-------------- scripts/backup/backup | 41 ++----- scripts/backup/decoy-trigger | 2 +- scripts/backup/monitor | 4 +- scripts/memory-usage | 29 ++--- scripts/start | 2 +- scripts/status/memory | 7 +- scripts/update/.updateignore | 1 - scripts/update/.updateinclude | 3 + services/electrum/electrs.yml | 2 +- services/manage.py | 15 +++ 28 files changed, 167 insertions(+), 199 deletions(-) create mode 100644 app/lib/composegenerator/next/stage1.py delete mode 100644 events/signals/.gitkeep rename events/triggers/{use-beta-builds => set-update-channel} (79%) delete mode 100755 events/triggers/use-stable-builds diff --git a/.gitignore b/.gitignore index 190b7c5..5e50f48 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ electrs/* fulcrumx/* nginx/* redis/* -events/signals/* docker-compose.override.yml # Commit these empty directories @@ -36,7 +35,6 @@ docker-compose.override.yml !db/citadel-seed db/citadel-seed/* !db/citadel-seed/.gitkeep -!events/signals/.gitkeep !lnd/.gitkeep !logs/.gitkeep !tor/data/.gitkeep @@ -46,9 +44,9 @@ db/citadel-seed/* !nginx/.gitkeep !redis/.gitkeep !fulcrumx/.gitkeep -!events/signals/.gitkeep !**/*.license services/installed.json services/installed.yml +events/karen diff --git a/app/lib/citadelutils.py b/app/lib/citadelutils.py index 0d00813..fcc4478 100644 --- a/app/lib/citadelutils.py +++ b/app/lib/citadelutils.py @@ -75,7 +75,7 @@ def is_builtin_type(obj): def classToDict(theClass): obj: dict = {} for key, value in theClass.__dict__.items(): - if value is None or (isinstance(value, list) and len(value) == 0): + if type(value).__name__ == "NoneType" or (isinstance(value, list) and len(value) == 0): continue if isinstance(value, list): newList = [] @@ -83,7 +83,8 @@ def classToDict(theClass): if is_builtin_type(element): newList.append(element) else: - newList.append(classToDict(element)) + if type(element).__name__ != "NoneType": + newList.append(classToDict(element)) obj[key] = newList elif isinstance(value, dict): newDict = {} @@ -95,8 +96,7 @@ def classToDict(theClass): obj[key] = newDict elif is_builtin_type(value): obj[key] = value - else: - #print(value) + elif type(value).__name__ != "NoneType": obj[key] = classToDict(value) return obj diff --git a/app/lib/composegenerator/next/stage1.py b/app/lib/composegenerator/next/stage1.py new file mode 100644 index 0000000..ab0b825 --- /dev/null +++ b/app/lib/composegenerator/next/stage1.py @@ -0,0 +1,24 @@ +from lib.citadelutils import classToDict +from lib.composegenerator.shared.env import validateEnv + +from lib.composegenerator.v3.types import App, generateApp +from lib.composegenerator.v3.generate import convertContainerPermissions + +def createCleanConfigFromV3(app: dict, nodeRoot: str): + parsedApp: App = generateApp(app) + for container in range(len(parsedApp.containers)): + # TODO: Make this dynamic and not hardcoded + if parsedApp.containers[container].requires and "c-lightning" in parsedApp.containers[container].requires: + parsedApp.containers[container] = None + parsedApp = convertContainerPermissions(parsedApp) + parsedApp = validateEnv(parsedApp) + finalApp = classToDict(parsedApp) + try: + finalApp['permissions'] = finalApp['metadata']['dependencies'] + except: + finalApp['permissions'] = [] + finalApp['id'] = finalApp['metadata']['id'] + del finalApp['metadata'] + # Set version of the cache file format + finalApp['version'] = "1" + return finalApp diff --git a/app/lib/composegenerator/shared/env.py b/app/lib/composegenerator/shared/env.py index c95cf12..69b4318 100644 --- a/app/lib/composegenerator/shared/env.py +++ b/app/lib/composegenerator/shared/env.py @@ -40,13 +40,14 @@ def validateEnvStringOrListorDict(env: Union[str, Union[list, dict]], existingEn def validateEnv(app: App): # For every container of the app, check if all env vars in the strings in environment are defined in env for container in app.containers: - if container.environment_allow: - existingEnv = container.environment_allow - del container.environment_allow - else: - existingEnv = [] - if container.environment: - validateEnvStringOrListorDict(container.command, existingEnv, app.metadata.id, container.name) - validateEnvStringOrListorDict(container.entrypoint, existingEnv, app.metadata.id, container.name) - validateEnvStringOrListorDict(container.environment, existingEnv, app.metadata.id, container.name) + if container is not None: + if container.environment_allow: + existingEnv = container.environment_allow + del container.environment_allow + else: + existingEnv = [] + if container.environment: + validateEnvStringOrListorDict(container.command, existingEnv, app.metadata.id, container.name) + validateEnvStringOrListorDict(container.entrypoint, existingEnv, app.metadata.id, container.name) + validateEnvStringOrListorDict(container.environment, existingEnv, app.metadata.id, container.name) return app diff --git a/app/lib/composegenerator/v2/utils/networking.py b/app/lib/composegenerator/v2/utils/networking.py index aedb781..709969c 100644 --- a/app/lib/composegenerator/v2/utils/networking.py +++ b/app/lib/composegenerator/v2/utils/networking.py @@ -47,7 +47,7 @@ def getContainerHiddenService( if isinstance(container.hiddenServicePorts, int): return getHiddenServiceString( "{} {}".format(metadata.name, container.name), - metadata.id, + metadata.id if isMainContainer else "{}-{}".format(metadata.id, container.name), container.hiddenServicePorts, containerIp, container.hiddenServicePorts, @@ -55,7 +55,7 @@ def getContainerHiddenService( elif isinstance(container.hiddenServicePorts, list): return getHiddenServiceMultiPort( "{} {}".format(metadata.name, container.name), - metadata.id, + metadata.id if isMainContainer else "{}-{}".format(metadata.id, container.name), containerIp, container.hiddenServicePorts, ) @@ -77,14 +77,14 @@ def getContainerHiddenService( else: additionalHiddenServices[key] = value for key, value in additionalHiddenServices.items(): - otherHiddenServices += "\n" if isinstance(value, int): otherHiddenServices += "# {} {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format( - metadata.name, container.name, key, metadata.id, container.name + metadata.name, container.name, key, metadata.id, key ) otherHiddenServices += "HiddenServicePort {} {}:{}".format( value, containerIp, value ) + otherHiddenServices += "\n" elif isinstance(value, list): otherHiddenServices += getHiddenServiceMultiPort( "{} {}".format(metadata.name, key), "{}-{}".format(metadata.id, key), containerIp, value diff --git a/app/lib/composegenerator/v3/generate.py b/app/lib/composegenerator/v3/generate.py index 13cc7d2..3e2789f 100644 --- a/app/lib/composegenerator/v3/generate.py +++ b/app/lib/composegenerator/v3/generate.py @@ -16,20 +16,21 @@ from lib.composegenerator.shared.const import permissions def convertContainerPermissions(app: App) -> App: for container in app.containers: - for permission in app.metadata.dependencies: - if isinstance(permission, str): - if permission in permissions(): - container.environment_allow.extend(permissions()[permission]['environment_allow']) - container.volumes.extend(permissions()[permission]['volumes']) - else: - print("Warning: app {} defines unknown permission {}".format(app.metadata.name, permission)) - else: - for subPermission in permission: - if subPermission in permissions(): - container.environment_allow.extend(permissions()[subPermission]['environment_allow']) - container.volumes.extend(permissions()[subPermission]['volumes']) + if container is not None: + for permission in app.metadata.dependencies: + if isinstance(permission, str): + if permission in permissions(): + container.environment_allow.extend(permissions()[permission]['environment_allow']) + container.volumes.extend(permissions()[permission]['volumes']) else: - print("Warning: app {} defines unknown permission {}".format(app.metadata.name, subPermission)) + print("Warning: app {} defines unknown permission {}".format(app.metadata.name, permission)) + else: + for subPermission in permission: + if subPermission in permissions(): + container.environment_allow.extend(permissions()[subPermission]['environment_allow']) + container.volumes.extend(permissions()[subPermission]['volumes']) + else: + print("Warning: app {} defines unknown permission {}".format(app.metadata.name, subPermission)) return app def convertDataDirToVolumeGen3(app: App) -> AppStage2: @@ -84,11 +85,12 @@ def createComposeConfigFromV3(app: dict, nodeRoot: str): newApp = configureIps(newApp, networkingFile, envFile) # This is validated earlier for container in newApp.containers: - container.ports = container.requiredPorts + for tcpPort in container.requiredPorts: + container.ports.append("{}:{}".format(tcpPort, tcpPort)) del container.requiredPorts for container in newApp.containers: for udpPort in container.requiredUdpPorts: - container.ports.append("{}/udp".format(udpPort)) + container.ports.append("{}:{}/udp".format(udpPort, udpPort)) del container.requiredUdpPorts newApp = configureMainPort(newApp, nodeRoot) newApp = configureHiddenServices(newApp, nodeRoot) diff --git a/app/lib/composegenerator/v3/networking.py b/app/lib/composegenerator/v3/networking.py index 8280c95..66d5a96 100644 --- a/app/lib/composegenerator/v3/networking.py +++ b/app/lib/composegenerator/v3/networking.py @@ -77,6 +77,8 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: # If it doesn't contain a :, it's the port itself if mainPort == False: mainPort = mainContainer.ports[0] + if mainPort.find(":") != -1: + mainPort = mainPort.split(":")[1] else: mainContainer.ports = [portToAppend] diff --git a/app/lib/composegenerator/v3/types.py b/app/lib/composegenerator/v3/types.py index 64c1051..a58199d 100644 --- a/app/lib/composegenerator/v3/types.py +++ b/app/lib/composegenerator/v3/types.py @@ -38,7 +38,7 @@ class Container: port: Union[int, None] = None requiredPorts: list = field(default_factory=list) requiredUdpPorts: list = field(default_factory=list) - preferredOutsidePort: Union[int, None] = None + #preferredOutsidePort: Union[int, None] = None requiresPort: Union[bool, None] = None environment: Union[dict, None] = None data: list = field(default_factory=list) diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 0399c4d..6db5672 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -5,6 +5,7 @@ import os import yaml +from lib.composegenerator.next.stage1 import createCleanConfigFromV3 from lib.composegenerator.v2.networking import getMainContainer from lib.composegenerator.v1.networking import getFreePort from lib.entropy import deriveEntropy @@ -40,6 +41,7 @@ def getAppRegistry(apps, app_path): app_metadata = [] for app in apps: app_yml_path = os.path.join(app_path, app, 'app.yml') + app_cache_path = os.path.join(app_path, app, 'app.cache.json') if os.path.isfile(app_yml_path): try: with open(app_yml_path, 'r') as f: @@ -57,6 +59,8 @@ def getAppRegistry(apps, app_path): getPortsOldApp(app_yml, app) else: getPortsV3App(app_yml, app) + with open(app_cache_path, 'w') as f: + json.dump(createCleanConfigFromV3(app_yml, os.path.dirname(app_path)), f) except Exception as e: print(e) print("App {} is invalid!".format(app)) diff --git a/app/lib/validate.py b/app/lib/validate.py index f822659..9dfe236 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -7,18 +7,18 @@ import yaml from jsonschema import validate import yaml +scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") + +with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f: + schemaVersion1 = yaml.safe_load(f) +with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: + schemaVersion2 = yaml.safe_load(f) +with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: + schemaVersion3 = yaml.safe_load(f) # Validates app data # Returns true if valid, false otherwise -scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") def validateApp(app: dict): - with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f: - schemaVersion1 = yaml.safe_load(f) - with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: - schemaVersion2 = yaml.safe_load(f) - with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: - schemaVersion3 = yaml.safe_load(f) - if 'version' in app and str(app['version']) == "1": try: validate(app, schemaVersion1) diff --git a/bin/lncli b/bin/lncli index 3de018f..67753c0 100755 --- a/bin/lncli +++ b/bin/lncli @@ -13,7 +13,7 @@ CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)" result=$(docker compose \ --file "${CITADEL_ROOT}/docker-compose.yml" \ --env-file "${CITADEL_ROOT}/.env" \ - exec lnd lncli "$@") + exec lightning lncli "$@") # We need to echo with quotes to preserve output formatting echo "$result" diff --git a/docker-compose.yml b/docker-compose.yml index a4c2073..60eb321 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: tor: container_name: tor - image: lncm/tor:0.4.6.8@sha256:c262923ffd0bd224a4a4123cf1c88eea11e2314566b7b7e8a1f77969deeb0208 + image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227 user: toruser restart: on-failure volumes: @@ -15,7 +15,7 @@ services: ipv4_address: $TOR_PROXY_IP app-tor: container_name: app-tor - image: lncm/tor:0.4.6.8@sha256:c262923ffd0bd224a4a4123cf1c88eea11e2314566b7b7e8a1f77969deeb0208 + image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227 user: toruser restart: on-failure volumes: @@ -26,7 +26,7 @@ services: ipv4_address: $APPS_TOR_IP app-2-tor: container_name: app-2-tor - image: lncm/tor:0.4.6.8@sha256:c262923ffd0bd224a4a4123cf1c88eea11e2314566b7b7e8a1f77969deeb0208 + image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227 user: toruser restart: on-failure volumes: @@ -37,7 +37,7 @@ services: ipv4_address: $APPS_2_TOR_IP app-3-tor: container_name: app-3-tor - image: lncm/tor:0.4.6.8@sha256:c262923ffd0bd224a4a4123cf1c88eea11e2314566b7b7e8a1f77969deeb0208 + image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227 user: toruser restart: on-failure volumes: @@ -100,7 +100,7 @@ services: ipv4_address: $LND_IP dashboard: container_name: dashboard - image: ghcr.io/runcitadel/dashboard:v0.0.12@sha256:102a44b938765be8224c63785949b90bf796670f01aef4e282d58a4c26a42595 + image: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5 restart: on-failure stop_grace_period: 1m30s networks: @@ -108,7 +108,7 @@ services: ipv4_address: $DASHBOARD_IP manager: container_name: manager - image: ghcr.io/runcitadel/manager:v0.0.12@sha256:de97eeb6e8caae5d1a8aeb045213621cf8e6e928177236bb0b3c25e76f6cd041 + image: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce depends_on: - tor - redis @@ -120,7 +120,7 @@ services: volumes: - ${PWD}/info.json:/info.json - ${PWD}/db:/db - - ${PWD}/events/signals:/signals + - ${PWD}/events:/events - ${PWD}/apps:/apps - ${PWD}/lnd:/lnd:ro - ${PWD}/statuses:/statuses @@ -162,7 +162,7 @@ services: ipv4_address: $MANAGER_IP middleware: container_name: middleware - image: ghcr.io/runcitadel/middleware:v0.0.10@sha256:afd6e2b6f5ba27cde32f6f6d630ddc6dd46d1072871f7834d7424d0554d0f53d + image: ghcr.io/runcitadel/middleware:v0.0.11@sha256:e472da8cbfa67d9a9dbf321334fe65cdf20a0f9b6d6bab33fdf07210f54e7002 depends_on: - manager - bitcoin diff --git a/events/signals/.gitkeep b/events/signals/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/events/triggers/app b/events/triggers/app index b1c987f..dee8381 100755 --- a/events/triggers/app +++ b/events/triggers/app @@ -6,8 +6,4 @@ CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" -signal="${1}" -command=${signal%%"-"*} -app=${signal#*"-"} - -"${CITADEL_ROOT}/scripts/app" "${command}" "${app}" +"${CITADEL_ROOT}/scripts/app" "$@" diff --git a/events/triggers/use-beta-builds b/events/triggers/set-update-channel similarity index 79% rename from events/triggers/use-beta-builds rename to events/triggers/set-update-channel index 252e9b5..38fdec5 100755 --- a/events/triggers/use-beta-builds +++ b/events/triggers/set-update-channel @@ -6,4 +6,4 @@ CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" -"${CITADEL_ROOT}/scripts/set-update-channel" beta +"${CITADEL_ROOT}/scripts/set-update-channel" "${1}" diff --git a/events/triggers/use-stable-builds b/events/triggers/use-stable-builds deleted file mode 100755 index 9a60e13..0000000 --- a/events/triggers/use-stable-builds +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" - -"${CITADEL_ROOT}/scripts/set-update-channel" stable diff --git a/info.json b/info.json index 5f36460..f35a1c9 100644 --- a/info.json +++ b/info.json @@ -1,6 +1,6 @@ { - "version": "0.0.3", - "name": "Citadel 0.0.3", + "version": "0.0.5", + "name": "Citadel 0.0.5", "requires": ">=0.0.1", - "notes": "This update fixes multiple bugs on the dashboard and contains a rewritten app system to prepare for Core Lightning compatibility. We also added a new dark mode toggle and updated the instructions to get a Lightning address." + "notes": "This update fixes a few bugs in the 0.0.4 release that were preventing some apps from working correctly." } diff --git a/karen b/karen index 28022a3..df99ea5 100755 --- a/karen +++ b/karen @@ -1,83 +1,27 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2020 Umbrel. https://getumbrel.com -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors. -# -# SPDX-License-Identifier: GPL-3.0-or-later +import socket +import os -# karen watches for signals and executes triggers in the events dir -# karen gets triggered a lot +rootDir = os.path.dirname(os.path.abspath(__file__)) +os.chdir(rootDir) -check_root () { - if [[ $UID != 0 ]]; then - echo "Error: This script must be run as root." - echo "Can I speak to a manager please?" - exit 1 - fi -} +if os.path.exists("events/karen"): + os.remove("events/karen") -check_if_not_already_running() { - if ps ax | grep $0 | grep -v $$ | grep bash | grep -v grep - then - echo "karen is already running" - exit 1 - fi -} +server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +server.bind("events/karen") -check_dependencies () { - for cmd in "$@"; do - if ! command -v $cmd >/dev/null 2>&1; then - echo "This script requires \"${cmd}\" to be installed" - exit 1 - fi - done -} - -check_root - -check_if_not_already_running - -check_dependencies fswatch readlink dirname - -if [[ -n "$1" ]]; then - root_dir="$(readlink -f $1)" -else - root_dir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))/events" -fi -signal_dir="$root_dir/signals" -trigger_dir="$root_dir/triggers" - -if [[ ! -d "$root_dir" ]]; then - echo "Root dir does not exist '$root_dir'" - exit 1 -fi - -echo "karen will start in 30 seconds" -sleep 30 -echo "karen is running in $root_dir" -fswatch -0 --event=PlatformSpecific $signal_dir | while read -d "" event; do - signal="${event#"$signal_dir"}" - signal="${signal#"/"}" - trigger="$trigger_dir/$signal" - args="" - - # If signal isn't an empty string, then it's a signal - if [[ -n "$signal" ]]; then - echo "Got signal: $signal" - - # If the signal starts wih app-, the trigger is "$trigger_dir/app" - # and the args are the rest of the signal - if [[ "$signal" =~ ^app- ]] && [[ "$signal" != "app-update" ]]; then - trigger="$trigger_dir/app" - args="${signal#"app-"}" - fi - if test -x "$trigger"; then - echo "karen is getting triggered!" - if [[ ! -f "statuses/update-in-progress" ]]; then - "$trigger" $args & - fi - else - echo "No trigger found" - fi - fi -done +while True: + server.listen(1) + conn, addr = server.accept() + datagram = conn.recv(1024) + if datagram: + instructions = datagram.decode("utf-8").strip().split() + cmd = instructions[0] + if(cmd == "trigger"): + trigger = instructions[1] + instructions.pop(0) + instructions.pop(0) + os.system("events/triggers/{} {}".format(trigger, " ".join(instructions))) + conn.close() diff --git a/scripts/backup/backup b/scripts/backup/backup index a0be32d..035716a 100755 --- a/scripts/backup/backup +++ b/scripts/backup/backup @@ -1,6 +1,7 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: 2020 Umbrel. https://getumbrel.com +# SPDX-FileCopyrightText: 2022 Citadel and contributors. https://runcitadel.space # # SPDX-License-Identifier: GPL-3.0-or-later @@ -98,39 +99,19 @@ tar \ # --verbose \ # --gzip -# Check if a file exists on Firebase by checking if -# "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F?alt=media" -# returns a 200 response code. -# -check_if_exists() { - curl -s -o /dev/null -w "%{http_code}" \ - -X GET \ - "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F${1}?alt=media" -} - -# Upload a file to Firebase Cloud Storage by uloading its name + a random ID. -# Before uploading, we need to check if a file with the same name already exists, and if it does, change the random ID. -# For example, if we want to upload a file named "" with the content AA, we'd use this command to upload: -# curl -X POST "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F?alt=media" -d "AA" -H "Content-Type: text/plain" -# To check if a file exists, we can check if this endpoint returns an error 404: -# curl "https://firebasestorage.googleapis.com/v0/b/citadel-user-backups.appspot.com/o/backups%2F?alt=media" -# To download a file, we can the same endpoint upload_file() { local file_to_send="${1}" - local file_name="${2}" - # A random ID to avoid collisions - local random_id="$(tr -dc A-Za-z0-9 /dev/null + "https://account.runcitadel.space/api/upload" \ + -d "${upload_data}" \ + -H "Content-Type: application/json" \ + --socks5 "localhost:${TOR_PROXY_PORT}" \ + > /dev/null } if [[ $BITCOIN_NETWORK == "testnet" ]]; then diff --git a/scripts/backup/decoy-trigger b/scripts/backup/decoy-trigger index 9d0465b..ddc8918 100755 --- a/scripts/backup/decoy-trigger +++ b/scripts/backup/decoy-trigger @@ -39,7 +39,7 @@ main () { echo "Sleeping for ${delay} seconds..." sleep $delay echo "Triggering decoy backup..." - touch "${CITADEL_ROOT}/events/signals/backup" + "${CITADEL_ROOT}/scripts/backup/backup" done } diff --git a/scripts/backup/monitor b/scripts/backup/monitor index 5234ac9..6bc499c 100755 --- a/scripts/backup/monitor +++ b/scripts/backup/monitor @@ -48,10 +48,10 @@ monitor_file () { sleep 1 done echo "$file_path created! Triggering backup..." - touch "${CITADEL_ROOT}/events/signals/backup" + "${CITADEL_ROOT}/scripts/backup/backup" fi - fswatch -0 --event Updated $file_path | xargs -0 -n 1 -I {} touch "${CITADEL_ROOT}/events/signals/backup" + fswatch -0 --event Updated $file_path | xargs -0 -n 1 -I {} "${CITADEL_ROOT}/scripts/backup/backup" } if [[ ! -d "${CITADEL_ROOT}" ]]; then diff --git a/scripts/memory-usage b/scripts/memory-usage index 1ff1f49..9c2ea19 100755 --- a/scripts/memory-usage +++ b/scripts/memory-usage @@ -21,7 +21,17 @@ function get_memory_usage() { function mem_usage_to_percent() { local mem_usage="$1" local total_mem="$(free -m | awk 'NR==2 {print $2}')" - echo "$(awk "BEGIN {printf \"%.1f\", $mem_usage / $total_mem * 100}")" + echo "$(awk "BEGIN {printf \"%.1f\", ${mem_usage/,/.} / ${total_mem/,/.} * 100}")" +} + +function app_mem_usage() { + # For every container of the app, get the mem usage, save it, and at the end, print the total mem usage of the app + local mem_usage=0 + for container in $(get_app_containers "$1"); do + # Use awk to add, it supports floating point numbers + mem_usage=$(awk "BEGIN {printf \"%.2f\", $mem_usage + $(get_memory_usage "$container")}") + done + echo "${1}: $mem_usage%" } get_total_used_mem_raw() { @@ -35,7 +45,7 @@ get_total_used_mem() { # To get the containers of the app, list every container whose name starts with the name of the app get_app_containers () { local app_name="$1" - "${CITADEL_ROOT}/scripts/app" compose "${app_name}" ps | awk '{print $1}' | grep -v 'Name\|-----' + "${CITADEL_ROOT}/scripts/app" compose "${app_name}" ps | awk '{print $1}' | grep -v 'NAME' } # Get the memory usage of the whole system, excluding docker containers @@ -48,22 +58,17 @@ get_system_memory_usage() { } main() { - echo "total: $(get_total_used_mem)%"& + echo "total: $(get_total_used_mem)%" + echo "system: $(get_system_memory_usage)%" for service in bitcoin lightning electrum tor; do echo "${service}: $(get_memory_usage "$service")%" & done for app in $("${CITADEL_ROOT}/scripts/app" ls-installed); do - # For every container of the app, get the mem usage, save it, and at the end, print the total mem usage of the app - local mem_usage=0 - for container in $(get_app_containers "$app"); do - # Use awk to add, it supports floating point numbers - mem_usage=$(awk "BEGIN {printf \"%.2f\", $mem_usage + $(get_memory_usage "$container")}") - done - wait - echo "${app}: $mem_usage%" + app_mem_usage "${app}" & done - echo "system: $(get_system_memory_usage)%" wait } +echo "Calculating memory usage..." +echo "This may take a while, please wait..." main | sort --key 2 --numeric-sort --reverse diff --git a/scripts/start b/scripts/start index 63eff7b..983a702 100755 --- a/scripts/start +++ b/scripts/start @@ -90,7 +90,7 @@ echo echo "Starting status monitors..." echo pkill -f ./scripts/status-monitor || true -./scripts/status-monitor memory 60 &>> "${CITADEL_LOGS}/status-monitor.log" & +./scripts/status-monitor memory 300 &>> "${CITADEL_LOGS}/status-monitor.log" & ./scripts/status-monitor storage 60 &>> "${CITADEL_LOGS}/status-monitor.log" & ./scripts/status-monitor temperature 15 &>> "${CITADEL_LOGS}/status-monitor.log" & ./scripts/status-monitor uptime 15 &>> "${CITADEL_LOGS}/status-monitor.log" & diff --git a/scripts/status/memory b/scripts/status/memory index 7ffe607..8b05f15 100755 --- a/scripts/status/memory +++ b/scripts/status/memory @@ -4,6 +4,9 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +# To prevent tools we use from outputting in another language +LANG=C + CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" memory_total_bytes() { @@ -23,7 +26,7 @@ get_app_memory_use() { local app_memory=0 - local app_containers=$("${CITADEL_ROOT}/app/app-manager.py" compose "${app}" ps | awk '{print $1}' | grep -v 'Name\|-----') + local app_containers=$("${CITADEL_ROOT}/scripts/app" compose "${app}" ps | awk '{print $1}' | grep -v 'NAME') for container in $app_containers; do local container_memory=$(get_container_memory_use "${container}") app_memory=$(awk "BEGIN {print ${app_memory}+${container_memory}}") @@ -43,7 +46,7 @@ used=$(memory_used_bytes) json=$(echo $json | jq --arg used "${used}" '. + {used: $used|tonumber}') cumulative_app_memory="0" -for app in $( "${CITADEL_ROOT}/app/app-manager.py" ls-installed); do +for app in $( "${CITADEL_ROOT}/scripts/app" ls-installed); do app_memory=$(get_app_memory_use "${app}") cumulative_app_memory=$(($cumulative_app_memory+$app_memory)) json=$(echo $json | jq --arg app "${app}" --arg app_memory "${app_memory}" '.breakdown |= .+ [{id: $app, used: $app_memory|tonumber}]') diff --git a/scripts/update/.updateignore b/scripts/update/.updateignore index b7915c6..92b78e1 100644 --- a/scripts/update/.updateignore +++ b/scripts/update/.updateignore @@ -5,7 +5,6 @@ lnd/* statuses tor/* electrs/* -events/signals logs/* app-data/* apps/networking.json diff --git a/scripts/update/.updateinclude b/scripts/update/.updateinclude index cfb9613..c3a164c 100644 --- a/scripts/update/.updateinclude +++ b/scripts/update/.updateinclude @@ -6,3 +6,6 @@ tor/torrc-apps-3 tor/torrc-core electrs/electrs.toml apps/docker-compose.common.yml +services/bitcoin/* +services/electrum/* +services/lightning/* diff --git a/services/electrum/electrs.yml b/services/electrum/electrs.yml index 9e63309..9873e3e 100644 --- a/services/electrum/electrs.yml +++ b/services/electrum/electrs.yml @@ -4,7 +4,7 @@ electrum: container_name: electrum - image: ghcr.io/runcitadel/electrs:v0.9.5@sha256:5fdd76415645de14f31c43844dc143b1477f86872d2f73a041c5005d469ed510 + image: ghcr.io/runcitadel/electrs:v0.9.6@sha256:cf08513b8aa6f081e42ea92c5903d520db2caedbc20aaccf70f844663bbc8646 working_dir: /data volumes: - ${PWD}/bitcoin:/bitcoin:ro diff --git a/services/manage.py b/services/manage.py index 3e9a3dc..c965b2d 100755 --- a/services/manage.py +++ b/services/manage.py @@ -31,6 +31,21 @@ args = parser.parse_args() # Function to install a service # To install it, read the service's YAML file (nodeRoot/services/name.yml) and add it to the main compose file (nodeRoot/docker-compose.yml) def setService(name, implementation): + # Get all available services + services = next(os.walk(os.path.join(nodeRoot, "services")))[1] + + if not name in services: + print("\"{}\" is not a valid service.".format(name)) + exit(1) + + # Get all available implementations + implementations = next(os.walk(os.path.join(nodeRoot, "services", name)), (None, None, []))[2] + implementations = [x.split('.')[0] for x in implementations] + + if not implementation in implementations: + print("\"{}\" is not a valid implementation.".format(implementation)) + exit(1) + # Read the YAML file with open(os.path.join(nodeRoot, "services", name, implementation + ".yml"), 'r') as stream: service = yaml.safe_load(stream)