From cea004770c6b3ac17f4e5e5fff4d3a7c9695815e Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sat, 16 Jul 2022 19:28:39 +0200 Subject: [PATCH 01/33] Implement quick updates (#56) Co-authored-by: nolim1t - f6287b82CC84bcbd Co-authored-by: Philipp Walter --- app-system/sources.list | 2 +- app/lib/composegenerator/next/stage1.py | 24 ------ app/lib/manage.py | 109 +++++++++++++++++++----- app/lib/metadata.py | 60 ++++++++----- app/lib/validate.py | 46 +++++----- db/dependencies.yml | 5 ++ docker-compose.yml | 7 +- events/triggers/quick-update | 25 ++++++ events/triggers/set-update-channel | 1 + info.json | 1 + scripts/configure | 39 ++++++--- scripts/set-update-channel | 2 +- 12 files changed, 219 insertions(+), 102 deletions(-) delete mode 100644 app/lib/composegenerator/next/stage1.py create mode 100644 db/dependencies.yml create mode 100755 events/triggers/quick-update diff --git a/app-system/sources.list b/app-system/sources.list index 2466e50..372320e 100644 --- a/app-system/sources.list +++ b/app-system/sources.list @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # A collection of fully FLOSS app definitions and FLOSS apps for Citadel. -https://github.com/runcitadel/apps v3-stable +https://github.com/runcitadel/apps v4-beta # Some apps modified version of Umbrel apps, and their app definitions aren't FLOSS yet. # Include them anyway, but as a separate repo. diff --git a/app/lib/composegenerator/next/stage1.py b/app/lib/composegenerator/next/stage1.py deleted file mode 100644 index ab0b825..0000000 --- a/app/lib/composegenerator/next/stage1.py +++ /dev/null @@ -1,24 +0,0 @@ -from lib.citadelutils import classToDict -from lib.composegenerator.shared.env import validateEnv - -from lib.composegenerator.v3.types import App, generateApp -from lib.composegenerator.v3.generate import convertContainerPermissions - -def createCleanConfigFromV3(app: dict, nodeRoot: str): - parsedApp: App = generateApp(app) - for container in range(len(parsedApp.containers)): - # TODO: Make this dynamic and not hardcoded - if parsedApp.containers[container].requires and "c-lightning" in parsedApp.containers[container].requires: - parsedApp.containers[container] = None - parsedApp = convertContainerPermissions(parsedApp) - parsedApp = validateEnv(parsedApp) - finalApp = classToDict(parsedApp) - try: - finalApp['permissions'] = finalApp['metadata']['dependencies'] - except: - finalApp['permissions'] = [] - finalApp['id'] = finalApp['metadata']['id'] - del finalApp['metadata'] - # Set version of the cache file format - finalApp['version'] = "1" - return finalApp diff --git a/app/lib/manage.py b/app/lib/manage.py index 37984f1..6e65efd 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -6,9 +6,11 @@ import stat import sys import tempfile import threading +import random from typing import List from sys import argv import os +import fcntl import requests import shutil import json @@ -32,9 +34,31 @@ from lib.validate import findAndValidateApps from lib.metadata import getAppRegistry from lib.entropy import deriveEntropy +class FileLock: + """Implements a file-based lock using flock(2). + The lock file is saved in directory dir with name lock_name. + dir is the current directory by default. + """ + + def __init__(self, lock_name, dir="."): + self.lock_file = open(os.path.join(dir, lock_name), "w") + + def acquire(self, blocking=True): + """Acquire the lock. + If the lock is not already acquired, return None. If the lock is + acquired and blocking is True, block until the lock is released. If + the lock is acquired and blocking is False, raise an IOError. + """ + ops = fcntl.LOCK_EX + if not blocking: + ops |= fcntl.LOCK_NB + fcntl.flock(self.lock_file, ops) + + def release(self): + """Release the lock. Return None even if lock not currently acquired""" + fcntl.flock(self.lock_file, fcntl.LOCK_UN) + # For an array of threads, join them and wait for them to finish - - def joinThreads(threads: List[threading.Thread]): for thread in threads: thread.join() @@ -50,26 +74,58 @@ updateIgnore = os.path.join(appsDir, ".updateignore") appDataDir = os.path.join(nodeRoot, "app-data") userFile = os.path.join(nodeRoot, "db", "user.json") legacyScript = os.path.join(nodeRoot, "scripts", "app") +with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file: + dependencies = yaml.safe_load(file) + # Returns a list of every argument after the second one in sys.argv joined into a string by spaces - - def getArguments(): arguments = "" for i in range(3, len(argv)): arguments += argv[i] + " " return arguments +def handleAppV4(app): + composeFile = os.path.join(appsDir, app, "docker-compose.yml") + os.chown(os.path.join(appsDir, app), 1000, 1000) + os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli convert --app-name '{}' --port-map /apps/ports.json /apps/{}/app.yml /apps/{}/result.yml --services 'lnd'".format(appsDir, dependencies['app-cli'], app, app, app)) + with open(os.path.join(appsDir, app, "result.yml"), "r") as resultFile: + resultYml = yaml.safe_load(resultFile) + with open(composeFile, "w") as dockerComposeFile: + yaml.dump(resultYml["spec"], dockerComposeFile) + torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] + torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] + with open(os.path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: + f.write(resultYml["new_tor_entries"]) + mainPort = resultYml["port"] + registryFile = os.path.join(nodeRoot, "apps", "registry.json") + registry: list = [] + lock = FileLock("citadeL_registry_lock", dir="/tmp") + lock.acquire() + if os.path.isfile(registryFile): + with open(registryFile, 'r') as f: + registry = json.load(f) + else: + raise Exception("Registry file not found") + + for registryApp in registry: + if registryApp['id'] == app: + registry[registry.index(registryApp)]['port'] = resultYml["port"] + break + + with open(registryFile, 'w') as f: + json.dump(registry, f, indent=4, sort_keys=True) + lock.release() def getAppYml(name): with open(os.path.join(appsDir, "sourceMap.json"), "r") as f: sourceMap = json.load(f) if not name in sourceMap: - print("Warning: App {} is not in the source map".format(name)) + print("Warning: App {} is not in the source map".format(name), file=sys.stderr) sourceMap = { name: { - "githubRepo": "runcitadel/core", - "branch": "v2" + "githubRepo": "runcitadel/apps", + "branch": "v4-stable" } } url = 'https://raw.githubusercontent.com/{}/{}/apps/{}/app.yml'.format(sourceMap[name]["githubRepo"], sourceMap[name]["branch"], name) @@ -89,16 +145,31 @@ def update(verbose: bool = False): json.dump(registry["ports"], f, sort_keys=True) print("Wrote registry to registry.json") + os.system("docker pull {}".format(dependencies['app-cli'])) + threads = list() # Loop through the apps and generate valid compose files from them, then put these into the app dir for app in apps: - composeFile = os.path.join(appsDir, app, "docker-compose.yml") - appYml = os.path.join(appsDir, app, "app.yml") - with open(composeFile, "w") as f: - appCompose = getApp(appYml, app) - if appCompose: - f.write(yaml.dump(appCompose, sort_keys=False)) - if verbose: - print("Wrote " + app + " to " + composeFile) + try: + composeFile = os.path.join(appsDir, app, "docker-compose.yml") + appYml = os.path.join(appsDir, app, "app.yml") + with open(appYml, 'r') as f: + appDefinition = yaml.safe_load(f) + if 'citadel_version' in appDefinition: + thread = threading.Thread(target=handleAppV4, args=(app,)) + thread.start() + threads.append(thread) + else: + appCompose = getApp(appDefinition, app) + with open(composeFile, "w") as f: + if appCompose: + f.write(yaml.dump(appCompose, sort_keys=False)) + if verbose: + print("Wrote " + app + " to " + composeFile) + except Exception as err: + print("Failed to convert app {}".format(app)) + print(err) + + joinThreads(threads) print("Generated configuration successfully") @@ -158,12 +229,7 @@ def stopInstalled(): joinThreads(threads) # Loads an app.yml and converts it to a docker-compose.yml - - -def getApp(appFile: str, appId: str): - with open(appFile, 'r') as f: - app = yaml.safe_load(f) - +def getApp(app, appId: str): if not "metadata" in app: raise Exception("Error: Could not find metadata in " + appFile) app["metadata"]["id"] = appId @@ -175,6 +241,7 @@ def getApp(appFile: str, appId: str): print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId)) return createComposeConfigFromV2(app, nodeRoot) elif 'version' in app and str(app['version']) == "3": + print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.3.0".format(appId)) return createComposeConfigFromV3(app, nodeRoot) else: raise Exception("Error: Unsupported version of app.yml") diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 6db5672..cf19672 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -4,10 +4,10 @@ import os import yaml +import traceback -from lib.composegenerator.next.stage1 import createCleanConfigFromV3 from lib.composegenerator.v2.networking import getMainContainer -from lib.composegenerator.v1.networking import getFreePort +from lib.composegenerator.shared.networking import assignIpV4 from lib.entropy import deriveEntropy from typing import List import json @@ -41,11 +41,15 @@ def getAppRegistry(apps, app_path): app_metadata = [] for app in apps: app_yml_path = os.path.join(app_path, app, 'app.yml') - app_cache_path = os.path.join(app_path, app, 'app.cache.json') if os.path.isfile(app_yml_path): try: with open(app_yml_path, 'r') as f: app_yml = yaml.safe_load(f.read()) + version = False + if 'version' in app_yml: + version = int(app_yml['version']) + elif 'citadel_version' in app_yml: + version = int(app_yml['citadel_version']) metadata: dict = app_yml['metadata'] metadata['id'] = app metadata['path'] = metadata.get('path', '') @@ -55,14 +59,14 @@ def getAppRegistry(apps, app_path): if "mainContainer" in metadata: metadata.pop("mainContainer") app_metadata.append(metadata) - if(app_yml["version"] != 3): + if version < 3: getPortsOldApp(app_yml, app) - else: + elif version == 3: getPortsV3App(app_yml, app) - with open(app_cache_path, 'w') as f: - json.dump(createCleanConfigFromV3(app_yml, os.path.dirname(app_path)), f) + elif version == 4: + getPortsV4App(app_yml, app) except Exception as e: - print(e) + print(traceback.format_exc()) print("App {} is invalid!".format(app)) appPortsToMap() return { @@ -97,12 +101,12 @@ def getNewPort(usedPorts): lastPort2 = lastPort2 + 1 return lastPort2 -def validatePort(appContainer, port, appId, priority: int, isDynamic = False): +def validatePort(containerName, appContainer, port, appId, priority: int, isDynamic = False): if port not in appPorts and port not in citadelPorts and port != 0: appPorts[port] = { "app": appId, "port": port, - "container": appContainer["name"], + "container": containerName, "priority": priority, "dynamic": isDynamic, } @@ -115,7 +119,7 @@ def validatePort(appContainer, port, appId, priority: int, isDynamic = False): appPorts[port] = { "app": appId, "port": port, - "container": appContainer["name"], + "container": containerName, "priority": priority, "dynamic": isDynamic, } @@ -128,7 +132,7 @@ def validatePort(appContainer, port, appId, priority: int, isDynamic = False): appPorts[newPort] = { "app": appId, "port": port, - "container": appContainer["name"], + "container": containerName, "priority": priority, "dynamic": isDynamic, } @@ -136,28 +140,44 @@ def validatePort(appContainer, port, appId, priority: int, isDynamic = False): def getPortsOldApp(app, appId): for appContainer in app["containers"]: if "port" in appContainer: - validatePort(appContainer, appContainer["port"], appId, 0) + validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0) if "ports" in appContainer: for port in appContainer["ports"]: realPort = int(str(port).split(":")[0]) - validatePort(appContainer, realPort, appId, 2) + validatePort(appContainer["name"], appContainer, realPort, appId, 2) def getPortsV3App(app, appId): for appContainer in app["containers"]: if "port" in appContainer: if "preferredOutsidePort" in appContainer and "requiresPort" in appContainer and appContainer["requiresPort"]: - validatePort(appContainer, appContainer["preferredOutsidePort"], appId, 2) + validatePort(appContainer["name"], appContainer, appContainer["preferredOutsidePort"], appId, 2) elif "preferredOutsidePort" in appContainer: - validatePort(appContainer, appContainer["preferredOutsidePort"], appId, 1) + validatePort(appContainer["name"], appContainer, appContainer["preferredOutsidePort"], appId, 1) else: - validatePort(appContainer, appContainer["port"], appId, 0) + validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0) elif "requiredPorts" not in appContainer and "requiredUdpPorts" not in appContainer: - validatePort(appContainer, getNewPort(appPorts.keys()), appId, 0, True) + validatePort(appContainer["name"], appContainer, getNewPort(appPorts.keys()), appId, 0, True) if "requiredPorts" in appContainer: for port in appContainer["requiredPorts"]: - validatePort(appContainer, port, appId, 2) + validatePort(appContainer["name"], appContainer, port, appId, 2) if "requiredUdpPorts" in appContainer: for port in appContainer["requiredUdpPorts"]: - validatePort(appContainer, port, appId, 2) \ No newline at end of file + validatePort(appContainer["name"], appContainer, port, appId, 2) + +def getPortsV4App(app, appId): + for appContainerName in app["services"].keys(): + appContainer = app["services"][appContainerName] + if "enable_networking" in appContainer and not appContainer["enable_networking"]: + return + assignIpV4(appId, appContainerName) + if "port" in appContainer: + validatePort(appContainerName, appContainer, appContainer["port"], appId, 0) + if "required_ports" in appContainer: + if "tcp" in appContainer["required_ports"]: + for port in appContainer["required_ports"]["tcp"].keys(): + validatePort(appContainerName, appContainer, port, appId, 2) + if "udp" in appContainer["required_ports"]: + for port in appContainer["required_ports"]["udp"].keys(): + validatePort(appContainerName, appContainer, port, appId, 2) diff --git a/app/lib/validate.py b/app/lib/validate.py index 9dfe236..5b2e4df 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -6,6 +6,7 @@ import os import yaml from jsonschema import validate import yaml +import traceback scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") @@ -33,7 +34,7 @@ def validateApp(app: dict): return True # Catch and log any errors, and return false except Exception as e: - print(e) + print(traceback.format_exc()) return False elif 'version' in app and str(app['version']) == "3": try: @@ -41,12 +42,13 @@ def validateApp(app: dict): return True # Catch and log any errors, and return false except Exception as e: - print(e) + print(traceback.format_exc()) return False - else: + elif 'version' not in app and 'citadel_version' not in app: print("Unsupported app version") return False - + else: + return True # Read in an app.yml file and pass it to the validation function # Returns true if valid, false otherwise @@ -72,14 +74,17 @@ def findApps(dir: str): def findAndValidateApps(dir: str): apps = [] app_data = {} - for root, dirs, files in os.walk(dir, topdown=False): - for name in dirs: - app_dir = os.path.join(root, name) - if os.path.isfile(os.path.join(app_dir, "app.yml")): - apps.append(name) - # Read the app.yml and append it to app_data - with open(os.path.join(app_dir, "app.yml"), 'r') as f: - app_data[name] = yaml.safe_load(f) + for subdir in os.scandir(dir): + if not subdir.is_dir(): + continue + app_dir = subdir.path + if os.path.isfile(os.path.join(app_dir, "app.yml")): + apps.append(subdir.name) + # Read the app.yml and append it to app_data + with open(os.path.join(app_dir, "app.yml"), 'r') as f: + app_data[subdir.name] = yaml.safe_load(f) + else: + print("App {} has no app.yml".format(subdir.name)) # Now validate all the apps using the validateAppFile function by passing the app.yml as an argument to it, if an app is invalid, remove it from the list for app in apps: appyml = app_data[app] @@ -113,12 +118,13 @@ def findAndValidateApps(dir: str): should_continue=False if not should_continue: continue - for container in appyml['containers']: - if 'permissions' in container: - for permission in container['permissions']: - if permission not in appyml['metadata']['dependencies'] and permission not in ["root", "hw"]: - print("WARNING: App {}'s container '{}' requires the '{}' permission, but the app doesn't list it in it's dependencies".format(app, container['name'], permission)) - apps.remove(app) - # Skip to the next iteration of the loop - continue + if 'containers' in appyml: + for container in appyml['containers']: + if 'permissions' in container: + for permission in container['permissions']: + if permission not in appyml['metadata']['dependencies'] and permission not in ["root", "hw"]: + print("WARNING: App {}'s container '{}' requires the '{}' permission, but the app doesn't list it in it's dependencies".format(app, container['name'], permission)) + apps.remove(app) + # Skip to the next iteration of the loop + continue return apps diff --git a/db/dependencies.yml b/db/dependencies.yml new file mode 100644 index 0000000..4decf53 --- /dev/null +++ b/db/dependencies.yml @@ -0,0 +1,5 @@ +compose: v2.6.0 +dashboard: ghcr.io/runcitadel/dashboard:main@sha256:25b6fb413c10f47e186309c8737926c241c0f2bec923b2c08dd837b828f14dbd +manager: ghcr.io/runcitadel/manager:main@sha256:db5775e986d53e762e43331540bb1c05a27b362da94d587c4a4591c981c00ee4 +middleware: ghcr.io/runcitadel/middleware:main@sha256:2fbbfb2e818bf0462f74a6aaab192881615ae018e6dcb62a50d05f82ec622cb0 +app-cli: ghcr.io/runcitadel/app-cli:main@sha256:f532923eac28cfac03579cbb440397bcf16c8730f291b39eeada8278331f7054 diff --git a/docker-compose.yml b/docker-compose.yml index b322aa6..38a7cd3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,7 @@ services: ipv4_address: $LND_IP dashboard: container_name: dashboard - image: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5 + image: ghcr.io/runcitadel/dashboard:main@sha256:25b6fb413c10f47e186309c8737926c241c0f2bec923b2c08dd837b828f14dbd restart: on-failure stop_grace_period: 1m30s networks: @@ -108,7 +108,7 @@ services: ipv4_address: $DASHBOARD_IP manager: container_name: manager - image: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce + image: ghcr.io/runcitadel/manager:main@sha256:db5775e986d53e762e43331540bb1c05a27b362da94d587c4a4591c981c00ee4 depends_on: - tor - redis @@ -162,7 +162,7 @@ services: ipv4_address: $MANAGER_IP middleware: container_name: middleware - image: ghcr.io/runcitadel/middleware:v0.0.11@sha256:e472da8cbfa67d9a9dbf321334fe65cdf20a0f9b6d6bab33fdf07210f54e7002 + image: ghcr.io/runcitadel/middleware:main@sha256:2fbbfb2e818bf0462f74a6aaab192881615ae018e6dcb62a50d05f82ec622cb0 depends_on: - manager - bitcoin @@ -223,6 +223,7 @@ services: ipv4_address: $ELECTRUM_IP redis: container_name: redis + user: 1000:1000 image: redis:7.0.0-bullseye@sha256:ad0705f2e2344c4b642449e658ef4669753d6eb70228d46267685045bf932303 working_dir: /data volumes: diff --git a/events/triggers/quick-update b/events/triggers/quick-update new file mode 100755 index 0000000..5386e88 --- /dev/null +++ b/events/triggers/quick-update @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" + +RELEASE=$(cat "$CITADEL_ROOT"/statuses/update-status.json | jq .updateTo -r) + +cat < "$CITADEL_ROOT"/statuses/update-status.json +{"state": "installing", "progress": 30, "description": "Starting update", "updateTo": "$RELEASE"} +EOF + +curl "https://raw.githubusercontent.com/runcitadel/core/${RELEASE}/db/dependencies.yml" > "$CITADEL_ROOT"/db/dependencies +cat < "$CITADEL_ROOT"/statuses/update-status.json +{"state": "installing", "progress": 70, "description": "Starting new containers", "updateTo": "$RELEASE"} +EOF + +"${CITADEL_ROOT}/scripts/start" + +cat < "$CITADEL_ROOT"/statuses/update-status.json +{"state": "success", "progress": 100, "description": "Successfully installed Citadel $RELEASE", "updateTo": ""} +EOF + diff --git a/events/triggers/set-update-channel b/events/triggers/set-update-channel index 38fdec5..3371cb4 100755 --- a/events/triggers/set-update-channel +++ b/events/triggers/set-update-channel @@ -7,3 +7,4 @@ CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" "${CITADEL_ROOT}/scripts/set-update-channel" "${1}" +"${CITADEL_ROOT}/scripts/start" diff --git a/info.json b/info.json index 6c4fc8c..3706fd6 100644 --- a/info.json +++ b/info.json @@ -2,5 +2,6 @@ "version": "0.0.7", "name": "Citadel 0.0.7", "requires": ">=0.0.1", + "isQuickUpdate": false, "notes": "While we are busy with the next huge update, you may need to wait longer for updates. This update updates Bitcoin Knots and LND to their latest versions to ensure apps can utilize their latest features. In addition, this update includes the Citadel CLI. More information on that will be published soon." } diff --git a/scripts/configure b/scripts/configure index 37ce08a..013ba91 100755 --- a/scripts/configure +++ b/scripts/configure @@ -31,13 +31,14 @@ if not is_arm64 and not is_amd64: print('Citadel only works on arm64 and amd64!') exit(1) +dependencies = False + # Check the output of "docker compose version", if it matches "Docker Compose version v2.0.0-rc.3", return true # Otherwise, return false -def is_compose_rc_or_outdated(): +def is_compose_version_except(target_version): try: output = subprocess.check_output(['docker', 'compose', 'version']) - if output.decode('utf-8').strip() != 'Docker Compose version v2.3.3': - print("Using outdated Docker Compose, updating...") + if output.decode('utf-8').strip() != 'Docker Compose version {}'.format(target_version): return True else: return False @@ -48,17 +49,19 @@ def is_compose_rc_or_outdated(): def download_docker_compose(): # Skip if os.path.expanduser('~/.docker/cli-plugins/docker-compose') exists subprocess.check_call(["mkdir", "-p", os.path.expanduser('~/.docker/cli-plugins/')]) - if (os.path.exists(os.path.expanduser('~/.docker/cli-plugins/docker-compose')) or os.path.exists('/usr/lib/docker/cli-plugins/docker-compose')) and not is_compose_rc_or_outdated(): - print("Found {}\n".format(subprocess.check_output(['docker', 'compose', 'version']).decode('utf-8').strip())) - return - - print("Installing Docker Compose...\n") - if is_arm64: - subprocess.check_call(['wget', 'https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-linux-aarch64', '-O', os.path.expanduser('~/.docker/cli-plugins/docker-compose')]) + compose_arch = 'aarch64' elif is_amd64: - subprocess.check_call(['wget', 'https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-linux-x86_64', '-O', os.path.expanduser('~/.docker/cli-plugins/docker-compose')]) - os.chmod(os.path.expanduser('~/.docker/cli-plugins/docker-compose'), 0o755) + compose_arch = 'x86_64' + # We validate that no other case than the two above can happen before + + if is_compose_version_except(dependencies['compose']): + print("Docker compose not found or not required version, updating.") + compose_url = 'https://github.com/docker/compose/releases/download/{}/docker-compose-linux-{}'.format(dependencies['compose'], compose_arch) + compose_file = os.path.expanduser('~/.docker/cli-plugins/docker-compose') + subprocess.check_call(['wget', compose_url, '-O', compose_file]) + os.chmod(compose_file, 0o755) + if not shutil.which("wget"): print('Wget is not installed!') @@ -72,6 +75,9 @@ if not shutil.which("docker"): CITADEL_ROOT=os.path.dirname(os.path.dirname(os.path.abspath(__file__))) os.chdir(CITADEL_ROOT) +with open("./db/dependencies.yml", "r") as file: + dependencies = yaml.safe_load(file) + updating = False status_dir = os.path.join(CITADEL_ROOT, 'statuses') # Make sure to use the main status dir for updates @@ -365,6 +371,15 @@ print("Generated configuration files\n") print("Checking if Docker Compose is installed...") download_docker_compose() +print("Updating core services...") +print() +with open("docker-compose.yml", 'r') as stream: + compose = yaml.safe_load(stream) +for service in ["manager", "middleware", "dashboard"]: + compose["services"][service]["image"] = dependencies[service] +with open("docker-compose.yml", "w") as stream: + yaml.dump(compose, stream, sort_keys=False) + if not reconfiguring: print("Updating apps...\n") os.system('./scripts/app --invoked-by-configure update') diff --git a/scripts/set-update-channel b/scripts/set-update-channel index a917095..1495702 100755 --- a/scripts/set-update-channel +++ b/scripts/set-update-channel @@ -10,7 +10,7 @@ NODE_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)" # If $1 is not given, fail if [ -z "$1" ]; then echo "Usage: $0 " - echo "Channel can currently either be 'stable' or 'beta'" + echo "Channel can currently either be 'stable', 'beta' or 'c-lightning'" exit 1 fi sed -i "s/UPDATE_CHANNEL=.*/UPDATE_CHANNEL=${1}/" "${NODE_ROOT}/.env" From 5496cc0baf5aa196a8efddca932f65dd820211cb Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sun, 17 Jul 2022 19:52:23 +0200 Subject: [PATCH 02/33] Citadel 0.1.0 Preview 1 (#67) --- info.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/info.json b/info.json index 3706fd6..f05b710 100644 --- a/info.json +++ b/info.json @@ -1,7 +1,7 @@ { - "version": "0.0.7", - "name": "Citadel 0.0.7", - "requires": ">=0.0.1", + "version": "0.1.0-preview.1", + "name": "Citadel 0.10 Preview 1", + "requires": ">=0.0.5", "isQuickUpdate": false, - "notes": "While we are busy with the next huge update, you may need to wait longer for updates. This update updates Bitcoin Knots and LND to their latest versions to ensure apps can utilize their latest features. In addition, this update includes the Citadel CLI. More information on that will be published soon." + "notes": "This update brings some of the new features for Citadel 0.1.0 and a lot of internal changes." } From 2654ee62171fe58cce730652fabc98ad21d9154c Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sun, 17 Jul 2022 21:22:21 +0200 Subject: [PATCH 03/33] Fix: 0.1.0 update (#68) --- scripts/update/.updateinclude | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/update/.updateinclude b/scripts/update/.updateinclude index c3a164c..f376610 100644 --- a/scripts/update/.updateinclude +++ b/scripts/update/.updateinclude @@ -9,3 +9,4 @@ apps/docker-compose.common.yml services/bitcoin/* services/electrum/* services/lightning/* +db/dependencies.yml From 6333cec0906b1d0599334a4eb55f28feaaa3d262 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sun, 17 Jul 2022 21:22:44 +0200 Subject: [PATCH 04/33] Update info.json --- info.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/info.json b/info.json index f05b710..f92fb6c 100644 --- a/info.json +++ b/info.json @@ -1,6 +1,6 @@ { - "version": "0.1.0-preview.1", - "name": "Citadel 0.10 Preview 1", + "version": "0.1.0-preview.2", + "name": "Citadel 0.10 Preview 2", "requires": ">=0.0.5", "isQuickUpdate": false, "notes": "This update brings some of the new features for Citadel 0.1.0 and a lot of internal changes." From 172faedc8df52c3902263a22480a02e1a5ca8664 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Wed, 20 Jul 2022 06:45:44 +0200 Subject: [PATCH 05/33] Preprocess app.yml.jinja files (#70) * Preprocess app.yml.jinja files * Fix preprocess * Update app cli * Try to fix permissions * Actually fix permissions --- app/lib/validate.py | 7 +++++++ db/dependencies.yml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/lib/validate.py b/app/lib/validate.py index 5b2e4df..05e2fb2 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -9,6 +9,7 @@ import yaml import traceback scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") +nodeRoot = os.path.join(scriptDir, "..") with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f: schemaVersion1 = yaml.safe_load(f) @@ -17,6 +18,9 @@ with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: schemaVersion3 = yaml.safe_load(f) +with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file: + dependencies = yaml.safe_load(file) + # Validates app data # Returns true if valid, false otherwise def validateApp(app: dict): @@ -78,6 +82,9 @@ def findAndValidateApps(dir: str): if not subdir.is_dir(): continue app_dir = subdir.path + if os.path.isfile(os.path.join(app_dir, "app.yml.jinja")): + os.chown(app_dir, 1000, 1000) + os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli preprocess --app-name '{}' --port-map /apps/ports.json /apps/{}/app.yml.jinja /apps/{}/app.yml --services 'lnd'".format(dir, dependencies['app-cli'], subdir.name, subdir.name, subdir.name)) if os.path.isfile(os.path.join(app_dir, "app.yml")): apps.append(subdir.name) # Read the app.yml and append it to app_data diff --git a/db/dependencies.yml b/db/dependencies.yml index 4decf53..751b3c3 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -2,4 +2,4 @@ compose: v2.6.0 dashboard: ghcr.io/runcitadel/dashboard:main@sha256:25b6fb413c10f47e186309c8737926c241c0f2bec923b2c08dd837b828f14dbd manager: ghcr.io/runcitadel/manager:main@sha256:db5775e986d53e762e43331540bb1c05a27b362da94d587c4a4591c981c00ee4 middleware: ghcr.io/runcitadel/middleware:main@sha256:2fbbfb2e818bf0462f74a6aaab192881615ae018e6dcb62a50d05f82ec622cb0 -app-cli: ghcr.io/runcitadel/app-cli:main@sha256:f532923eac28cfac03579cbb440397bcf16c8730f291b39eeada8278331f7054 +app-cli: ghcr.io/runcitadel/app-cli:main@sha256:79a99263643b129ccbc2a09d48d4820ab97c04d72c2f986daa6cb544474a54ad From 7393f846b3d996cd517bf4157581d7d8852f71a9 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sun, 7 Aug 2022 14:44:03 +0200 Subject: [PATCH 06/33] Update sources.list (#71) --- app-system/sources.list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app-system/sources.list b/app-system/sources.list index 372320e..c87e4b4 100644 --- a/app-system/sources.list +++ b/app-system/sources.list @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # A collection of fully FLOSS app definitions and FLOSS apps for Citadel. -https://github.com/runcitadel/apps v4-beta +https://github.com/runcitadel/apps v4-tmpl # Some apps modified version of Umbrel apps, and their app definitions aren't FLOSS yet. # Include them anyway, but as a separate repo. From f1cfd8df43c5d3b78f562ba50cd86a4e44a10d45 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Thu, 18 Aug 2022 11:45:59 +0000 Subject: [PATCH 07/33] Preview 3 --- info.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/info.json b/info.json index f92fb6c..1dcac7b 100644 --- a/info.json +++ b/info.json @@ -1,6 +1,6 @@ { - "version": "0.1.0-preview.2", - "name": "Citadel 0.10 Preview 2", + "version": "0.1.0-preview.3", + "name": "Citadel 0.10 Preview 3", "requires": ">=0.0.5", "isQuickUpdate": false, "notes": "This update brings some of the new features for Citadel 0.1.0 and a lot of internal changes." From b05e16515cde7ca65e145613a4b90e7de95427fd Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 23 Aug 2022 19:01:40 +0000 Subject: [PATCH 08/33] Remove unused function --- app/lib/citadelutils.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/app/lib/citadelutils.py b/app/lib/citadelutils.py index fcc4478..e0b044e 100644 --- a/app/lib/citadelutils.py +++ b/app/lib/citadelutils.py @@ -44,29 +44,6 @@ def parse_dotenv(file_path): exit(1) return envVars -# Combines an object and a class -# If the key exists in both objects, the value of the second object is used -# If the key does not exist in the first object, the value from the second object is used -# If a key contains a list, the second object's list is appended to the first object's list -# If a key contains another object, these objects are combined -def combineObjectAndClass(theClass, obj: dict): - for key, value in obj.items(): - if key in theClass.__dict__: - if isinstance(value, list): - if isinstance(theClass.__dict__[key], list): - theClass.__dict__[key].extend(value) - else: - theClass.__dict__[key] = [theClass.__dict__[key]] + value - elif isinstance(value, dict): - if isinstance(theClass.__dict__[key], dict): - theClass.__dict__[key].update(value) - else: - theClass.__dict__[key] = {theClass.__dict__[key]: value} - else: - theClass.__dict__[key] = value - else: - theClass.__dict__[key] = value - def is_builtin_type(obj): return isinstance(obj, (int, float, str, bool, list, dict)) From 926e863cefcfdbbc15f593433baef6ffb7781bd5 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 23 Aug 2022 19:07:33 +0000 Subject: [PATCH 09/33] Update all dependencies --- db/dependencies.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/db/dependencies.yml b/db/dependencies.yml index 751b3c3..dac2e22 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -1,5 +1,5 @@ -compose: v2.6.0 -dashboard: ghcr.io/runcitadel/dashboard:main@sha256:25b6fb413c10f47e186309c8737926c241c0f2bec923b2c08dd837b828f14dbd -manager: ghcr.io/runcitadel/manager:main@sha256:db5775e986d53e762e43331540bb1c05a27b362da94d587c4a4591c981c00ee4 -middleware: ghcr.io/runcitadel/middleware:main@sha256:2fbbfb2e818bf0462f74a6aaab192881615ae018e6dcb62a50d05f82ec622cb0 -app-cli: ghcr.io/runcitadel/app-cli:main@sha256:79a99263643b129ccbc2a09d48d4820ab97c04d72c2f986daa6cb544474a54ad +compose: v2.10.0 +dashboard: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e +manager: ghcr.io/runcitadel/manager:v0.0.17@sha256:ba436a07d6f96282217851756d8c81aeaa03c42dfa2246a89a78fc3384eed3cb +middleware: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae +app-cli: ghcr.io/runcitadel/app-cli:main@sha256:6dad26faf652b930cb219a6261af8edfa84d5db4d679153700dbc1b136267bbf From b7b76338f355e6a9b8089f0c0c5f346faa163563 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 23 Aug 2022 19:20:50 +0000 Subject: [PATCH 10/33] Add basic support for virtual apps --- app/lib/manage.py | 2 ++ app/lib/metadata.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/app/lib/manage.py b/app/lib/manage.py index 6e65efd..98ed977 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -143,6 +143,8 @@ def update(verbose: bool = False): json.dump(registry["metadata"], f, sort_keys=True) with open(os.path.join(appsDir, "ports.json"), "w") as f: json.dump(registry["ports"], f, sort_keys=True) + with open(os.path.join(appsDir, "virtual-apps.json"), "w") as f: + json.dump(registry["virtual_apps"], f, sort_keys=True) print("Wrote registry to registry.json") os.system("docker pull {}".format(dependencies['app-cli'])) diff --git a/app/lib/metadata.py b/app/lib/metadata.py index cf19672..69b0f1e 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -39,6 +39,7 @@ def appPortsToMap(): # Return a list of all app's metadata def getAppRegistry(apps, app_path): app_metadata = [] + virtual_apps = {} for app in apps: app_yml_path = os.path.join(app_path, app, 'app.yml') if os.path.isfile(app_yml_path): @@ -58,6 +59,11 @@ def getAppRegistry(apps, app_path): metadata['defaultPassword'] = deriveEntropy("app-{}-seed".format(app)) if "mainContainer" in metadata: metadata.pop("mainContainer") + if "implements" in metadata: + implements = metadata["implements"] + if implements not in virtual_apps: + virtual_apps[implements] = [] + virtual_apps[implements].append(app) app_metadata.append(metadata) if version < 3: getPortsOldApp(app_yml, app) @@ -70,6 +76,7 @@ def getAppRegistry(apps, app_path): print("App {} is invalid!".format(app)) appPortsToMap() return { + "virtual_apps": virtual_apps, "metadata": app_metadata, "ports": appPortMap } From e209f9f8cb1c2ef60088f0867a700b20a04a88ab Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Wed, 24 Aug 2022 11:25:53 +0000 Subject: [PATCH 11/33] Remove services argument for app compiler --- app/lib/manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index 98ed977..37f9c57 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -88,7 +88,7 @@ def getArguments(): def handleAppV4(app): composeFile = os.path.join(appsDir, app, "docker-compose.yml") os.chown(os.path.join(appsDir, app), 1000, 1000) - os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli convert --app-name '{}' --port-map /apps/ports.json /apps/{}/app.yml /apps/{}/result.yml --services 'lnd'".format(appsDir, dependencies['app-cli'], app, app, app)) + os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli convert --app-name '{}' --port-map /apps/ports.json /apps/{}/app.yml /apps/{}/result.yml".format(appsDir, dependencies['app-cli'], app, app, app)) with open(os.path.join(appsDir, app, "result.yml"), "r") as resultFile: resultYml = yaml.safe_load(resultFile) with open(composeFile, "w") as dockerComposeFile: From 61cbdb84785ba619606d1a4f69ce7627632e4a26 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Wed, 24 Aug 2022 11:27:55 +0000 Subject: [PATCH 12/33] Remove another unused argument --- app/lib/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/validate.py b/app/lib/validate.py index 05e2fb2..db3d5e0 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -84,7 +84,7 @@ def findAndValidateApps(dir: str): app_dir = subdir.path if os.path.isfile(os.path.join(app_dir, "app.yml.jinja")): os.chown(app_dir, 1000, 1000) - os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli preprocess --app-name '{}' --port-map /apps/ports.json /apps/{}/app.yml.jinja /apps/{}/app.yml --services 'lnd'".format(dir, dependencies['app-cli'], subdir.name, subdir.name, subdir.name)) + os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli preprocess --app-name '{}' /apps/{}/app.yml.jinja /apps/{}/app.yml --services 'lnd'".format(dir, dependencies['app-cli'], subdir.name, subdir.name, subdir.name)) if os.path.isfile(os.path.join(app_dir, "app.yml")): apps.append(subdir.name) # Read the app.yml and append it to app_data From eb2a347b43a003bb16521d968f092468a8a97016 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Wed, 24 Aug 2022 11:54:36 +0000 Subject: [PATCH 13/33] Citadel 0.0.8 --- info.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/info.json b/info.json index 1dcac7b..2fdc1b9 100644 --- a/info.json +++ b/info.json @@ -1,7 +1,7 @@ { - "version": "0.1.0-preview.3", - "name": "Citadel 0.10 Preview 3", + "version": "0.0.8", + "name": "Citadel 0.0.8", "requires": ">=0.0.5", "isQuickUpdate": false, - "notes": "This update brings some of the new features for Citadel 0.1.0 and a lot of internal changes." + "notes": "This update includes a new version of the Citadel app system and includes a few new apps. With the new app system, Citadel can handle more apps and makes app development eaier." } From 7ad22665e46977f553f88df18cc0a00dcbc6e554 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Wed, 24 Aug 2022 11:55:04 +0000 Subject: [PATCH 14/33] Use stable app repos --- app-system/sources.list | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app-system/sources.list b/app-system/sources.list index c87e4b4..77c85f0 100644 --- a/app-system/sources.list +++ b/app-system/sources.list @@ -3,9 +3,9 @@ # SPDX-License-Identifier: GPL-3.0-or-later # A collection of fully FLOSS app definitions and FLOSS apps for Citadel. -https://github.com/runcitadel/apps v4-tmpl +https://github.com/runcitadel/apps v4-stable # Some apps modified version of Umbrel apps, and their app definitions aren't FLOSS yet. # Include them anyway, but as a separate repo. # Add a # to the line below to disable the repo and only use FLOSS apps. -https://github.com/runcitadel/apps-nonfree v3-stable +https://github.com/runcitadel/apps-nonfree v4-stable From 34641d62e0bf2f39d1e0db338ad9f9f2e4c1a0dd Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Wed, 24 Aug 2022 12:08:12 +0000 Subject: [PATCH 15/33] Fix a typo --- app/lib/manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index 37f9c57..ea41d1e 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -100,7 +100,7 @@ def handleAppV4(app): mainPort = resultYml["port"] registryFile = os.path.join(nodeRoot, "apps", "registry.json") registry: list = [] - lock = FileLock("citadeL_registry_lock", dir="/tmp") + lock = FileLock("citadel_registry_lock", dir="/tmp") lock.acquire() if os.path.isfile(registryFile): with open(registryFile, 'r') as f: From 4133a8ae5ab2f5465b9036c2caaee34c078b0a54 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Wed, 24 Aug 2022 12:09:43 +0000 Subject: [PATCH 16/33] Update docker-compose.yml to match dependencies.yml --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 38a7cd3..eef9bf3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,7 @@ services: ipv4_address: $LND_IP dashboard: container_name: dashboard - image: ghcr.io/runcitadel/dashboard:main@sha256:25b6fb413c10f47e186309c8737926c241c0f2bec923b2c08dd837b828f14dbd + image: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e restart: on-failure stop_grace_period: 1m30s networks: @@ -108,7 +108,7 @@ services: ipv4_address: $DASHBOARD_IP manager: container_name: manager - image: ghcr.io/runcitadel/manager:main@sha256:db5775e986d53e762e43331540bb1c05a27b362da94d587c4a4591c981c00ee4 + image: ghcr.io/runcitadel/manager:v0.0.17@sha256:ba436a07d6f96282217851756d8c81aeaa03c42dfa2246a89a78fc3384eed3cb depends_on: - tor - redis @@ -162,7 +162,7 @@ services: ipv4_address: $MANAGER_IP middleware: container_name: middleware - image: ghcr.io/runcitadel/middleware:main@sha256:2fbbfb2e818bf0462f74a6aaab192881615ae018e6dcb62a50d05f82ec622cb0 + image: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae depends_on: - manager - bitcoin From 668a4ce55ac621e5d2f4011df27599b73b9167fe Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 30 Aug 2022 07:09:47 +0000 Subject: [PATCH 17/33] Deno manager --- db/dependencies.yml | 2 +- docker-compose.yml | 3 +-- scripts/update/01-run.sh | 8 +++----- templates/nginx-sample.conf | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/db/dependencies.yml b/db/dependencies.yml index dac2e22..cf5b99d 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -1,5 +1,5 @@ compose: v2.10.0 dashboard: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e -manager: ghcr.io/runcitadel/manager:v0.0.17@sha256:ba436a07d6f96282217851756d8c81aeaa03c42dfa2246a89a78fc3384eed3cb +manager: ghcr.io/runcitadel/manager:deno@sha256:38ef8474cc501d3f3e9ea63e73d1c48f848662467ffe5f7f0b9bbb44e04055cf middleware: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae app-cli: ghcr.io/runcitadel/app-cli:main@sha256:6dad26faf652b930cb219a6261af8edfa84d5db4d679153700dbc1b136267bbf diff --git a/docker-compose.yml b/docker-compose.yml index eef9bf3..2a6bfd2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,7 +108,7 @@ services: ipv4_address: $DASHBOARD_IP manager: container_name: manager - image: ghcr.io/runcitadel/manager:v0.0.17@sha256:ba436a07d6f96282217851756d8c81aeaa03c42dfa2246a89a78fc3384eed3cb + image: ghcr.io/runcitadel/manager:deno@sha256:38ef8474cc501d3f3e9ea63e73d1c48f848662467ffe5f7f0b9bbb44e04055cf depends_on: - tor - redis @@ -128,7 +128,6 @@ services: - jwt-public-key:/jwt-public-key - jwt-private-key:/jwt-private-key environment: - PORT: '3006' USER_PASSWORD_FILE: /db/user.json JWT_PUBLIC_KEY_FILE: /jwt-public-key/jwt.pem JWT_PRIVATE_KEY_FILE: /jwt-private-key/jwt.key diff --git a/scripts/update/01-run.sh b/scripts/update/01-run.sh index a3c5dce..f62456c 100755 --- a/scripts/update/01-run.sh +++ b/scripts/update/01-run.sh @@ -148,11 +148,9 @@ for app in $("$CITADEL_ROOT/scripts/app" ls-installed); do done wait -# If CITADEL_ROOT doesn't contain services/installed.json, then put '["electrs"]' into it. -# This is to ensure that the 0.5.0 update doesn't remove electrs. -if [[ ! -f "${CITADEL_ROOT}/services/installed.json" ]]; then - echo '["electrs"]' > "${CITADEL_ROOT}/services/installed.json" -fi +# Remove the nginx config (only for 0.0.9) +# So it will be recreated +rm -f nginx/nginx.conf # Start updated containers echo "Starting new containers" diff --git a/templates/nginx-sample.conf b/templates/nginx-sample.conf index 660785e..4fa1711 100644 --- a/templates/nginx-sample.conf +++ b/templates/nginx-sample.conf @@ -27,7 +27,7 @@ http { } location /manager-api/ { - proxy_pass http://:3006/; + proxy_pass http://:3000/; } # dashboard (old) From 33463391765c4ec503a44fa16fb8a0dd4d61e49b Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 30 Aug 2022 07:13:42 +0000 Subject: [PATCH 18/33] Try to fix port change issue --- app/lib/metadata.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 69b0f1e..524b19f 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -102,9 +102,11 @@ citadelPorts = [ lastPort = 3000 -def getNewPort(usedPorts): +def getNewPort(appPorts, appId): lastPort2 = lastPort - while lastPort2 in usedPorts or lastPort2 in citadelPorts: + while lastPort2 in appPorts.keys() or lastPort2 in citadelPorts: + if lastPort2 in appPorts.keys() and appPorts[lastPort2]["app"] == appId: + return lastPort2 lastPort2 = lastPort2 + 1 return lastPort2 @@ -119,7 +121,7 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna } else: if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != appContainer["name"]: - newPort = getNewPort(appPorts.keys()) + newPort = getNewPort(appPorts.keys(), appId) if port in appPorts and priority > appPorts[port]["priority"]: #print("Prioritizing app {} over {}".format(appId, appPorts[port]["app"])) appPorts[newPort] = appPorts[port].copy() From 5b1051fb4579274f86b5fd85432b111d35891acb Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 30 Aug 2022 07:19:26 +0000 Subject: [PATCH 19/33] Bug fixes --- app/lib/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 524b19f..8f5bc11 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -121,7 +121,7 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna } else: if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != appContainer["name"]: - newPort = getNewPort(appPorts.keys(), appId) + newPort = getNewPort(appPorts, appId) if port in appPorts and priority > appPorts[port]["priority"]: #print("Prioritizing app {} over {}".format(appId, appPorts[port]["app"])) appPorts[newPort] = appPorts[port].copy() @@ -167,7 +167,7 @@ def getPortsV3App(app, appId): else: validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0) elif "requiredPorts" not in appContainer and "requiredUdpPorts" not in appContainer: - validatePort(appContainer["name"], appContainer, getNewPort(appPorts.keys()), appId, 0, True) + validatePort(appContainer["name"], appContainer, getNewPort(appPorts, appId), appId, 0, True) if "requiredPorts" in appContainer: for port in appContainer["requiredPorts"]: validatePort(appContainer["name"], appContainer, port, appId, 2) From b066a7574d066358e123cedcc33907bbc4caba7d Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 30 Aug 2022 07:23:43 +0000 Subject: [PATCH 20/33] Better logging --- app/lib/manage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index ea41d1e..1ae8c0f 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -16,6 +16,7 @@ import shutil import json import yaml import subprocess +import traceback try: import semver except Exception: @@ -169,7 +170,7 @@ def update(verbose: bool = False): print("Wrote " + app + " to " + composeFile) except Exception as err: print("Failed to convert app {}".format(app)) - print(err) + print(traceback.format_exc()) joinThreads(threads) print("Generated configuration successfully") From 39be35fc929533dd8620101cbfda9550b1eeffa9 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sun, 19 Jun 2022 02:05:22 +0200 Subject: [PATCH 21/33] App system cleanups (#51) * Add app cli to docker-compose.yml * Remove app.yml v1 * Add missing import * More cleanups * Another missing import * Add mount for apps * Remove more --- app-standard-v1.json | 230 ------------------ app-standard-v1.json.license | 3 - app/app-standard-v1.yml | 194 --------------- app/lib/composegenerator/shared/env.py | 2 +- app/lib/composegenerator/shared/main.py | 2 +- app/lib/composegenerator/shared/networking.py | 137 +++++++++++ app/lib/composegenerator/v1/generate.py | 30 --- app/lib/composegenerator/v1/networking.py | 226 ----------------- app/lib/composegenerator/v1/types.py | 151 ------------ .../composegenerator/v1/utils/networking.py | 118 --------- app/lib/composegenerator/v2/networking.py | 2 +- app/lib/composegenerator/v3/networking.py | 4 +- app/lib/manage.py | 6 +- app/lib/validate.py | 12 +- docker-compose.yml | 7 + 15 files changed, 150 insertions(+), 974 deletions(-) delete mode 100644 app-standard-v1.json delete mode 100644 app-standard-v1.json.license delete mode 100644 app/app-standard-v1.yml create mode 100644 app/lib/composegenerator/shared/networking.py delete mode 100644 app/lib/composegenerator/v1/generate.py delete mode 100644 app/lib/composegenerator/v1/networking.py delete mode 100644 app/lib/composegenerator/v1/types.py delete mode 100644 app/lib/composegenerator/v1/utils/networking.py diff --git a/app-standard-v1.json b/app-standard-v1.json deleted file mode 100644 index 0ecb458..0000000 --- a/app-standard-v1.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Citadel app.yml v1", - "description": "The first draft of Citadel's app.yml format", - "type": "object", - "properties": { - "version": { - "type": [ - "string", - "number" - ], - "description": "The version of the app.yml format you're using." - }, - "metadata": { - "type": "object", - "properties": { - "name": { - "description": "Displayed name of the app", - "type": "string" - }, - "version": { - "description": "Displayed version for the app", - "type": "string" - }, - "category": { - "description": "The category you'd put the app in", - "type": "string" - }, - "tagline": { - "description": "A clever tagline", - "type": "string" - }, - "description": { - "description": "A longer description of the app", - "type": "string" - }, - "developer": { - "description": "The awesome people behind the app", - "type": "string" - }, - "website": { - "description": "Displayed version for the app", - "type": "string" - }, - "dependencies": { - "description": "The services the app depends on", - "type": "array", - "items": { - "type": "string" - } - }, - "repo": { - "description": "The development repository for your app", - "type": "string" - }, - "support": { - "description": "A link to the app support wiki/chat/...", - "type": "string" - }, - "gallery": { - "type": "array", - "description": "URLs or paths in the runcitadel/app-images/[app-name] folder with app images", - "items": { - "type": "string" - } - }, - "path": { - "description": "The path of the app's visible site the open button should open", - "type": "string" - }, - "defaultPassword": { - "description": "The app's default password", - "type": "string" - }, - "torOnly": { - "description": "Whether the app is only available over tor", - "type": "boolean" - }, - "mainContainer": { - "type": "string", - "description": "The name of the main container for the app. If set, IP, port, and hidden service will be assigned to it automatically." - }, - "updateContainer": { - "type": "string", - "description": "The container the developer system should automatically update." - } - }, - "required": [ - "name", - "version", - "category", - "tagline", - "description", - "developer", - "website", - "repo", - "support", - "gallery" - ], - "additionalProperties": false - }, - "containers": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "image": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "lnd", - "bitcoind", - "electrum", - "root", - "hw" - ] - } - }, - "ports": { - "type": "array", - "items": { - "type": [ - "string", - "number" - ] - } - }, - "port": { - "type": "number", - "description": "If this is the main container, the port inside the container which will be exposed to the outside as the port specified in metadata." - }, - "environment": { - "type": "object" - }, - "data": { - "type": "array", - "description": "An array of at directories in the container the app stores its data in. Can be empty. Please only list top-level directories.", - "items": { - "type": "string" - } - }, - "user": { - "type": "string", - "description": "The user the container should run as" - }, - "stop_grace_period": { - "type": "string", - "description": "The grace period for stopping the container. Defaults to 1 minute." - }, - "depends_on": { - "type": "array", - "description": "The services the container depends on" - }, - "entrypoint": { - "type": [ - "string", - "array" - ], - "description": "The entrypoint for the container" - }, - "bitcoin_mount_dir": { - "type": "string", - "description": "Where to mount the bitcoin dir" - }, - "command": { - "type": [ - "string", - "array" - ], - "description": "The command for the container" - }, - "init": { - "type": "boolean", - "description": "Whether the container should be run with init" - }, - "stop_signal": { - "type": "string", - "description": "The signal to send to the container when stopping" - }, - "noNetwork": { - "type": "boolean", - "description": "Set this to true if the container shouldn't get an IP & port exposed." - }, - "needsHiddenService": { - "type": "boolean", - "description": "Set this to true if the container should be assigned a hidden service even if it's not the main container." - }, - "hiddenServicePort": { - "type": "number", - "description": "Set this to a port if your container exposes multiple ports, but only one should be a hidden service." - }, - "hiddenServicePorts": { - "type": "object", - "description": "Set this to a map of service names to hidden service ports if your container exposes multiple ports, and all of them should be hidden services.", - "patternProperties": { - "^[a-zA-Z0-9_]+$": { - "type": [ - "number", - "array" - ] - } - } - }, - "restart": { - "type": "string", - "description": "When the container should restart. Can be 'always' or 'on-failure'." - } - }, - "additionalProperties": false, - "required": [ - "name", - "image" - ] - }, - "additionalProperties": false - } - }, - "required": [ - "metadata", - "containers" - ], - "additionalProperties": false -} diff --git a/app-standard-v1.json.license b/app-standard-v1.json.license deleted file mode 100644 index f0626d9..0000000 --- a/app-standard-v1.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2021 Citadel and contributors - -SPDX-License-Identifier: GPL-3.0-or-later \ No newline at end of file diff --git a/app/app-standard-v1.yml b/app/app-standard-v1.yml deleted file mode 100644 index 3000a25..0000000 --- a/app/app-standard-v1.yml +++ /dev/null @@ -1,194 +0,0 @@ -# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema -$schema: https://json-schema.org/draft/2020-12/schema - - -title: Citadel app.yml v1 -description: The first draft of Citadel's app.yml format -type: object - -properties: - version: - type: - - string - - number - description: The version of the app.yml format you're using. - - metadata: - type: object - properties: - name: - description: Displayed name of the app - type: string - version: - description: Displayed version for the app - type: string - category: - description: The category you'd put the app in - type: string - tagline: - description: A clever tagline - type: string - description: - description: A longer description of the app - type: string - developer: - description: The awesome people behind the app - type: string - website: - description: Displayed version for the app - type: string - dependencies: - description: The services the app depends on - type: array - items: - type: string - repo: - description: The development repository for your app - type: string - support: - description: A link to the app support wiki/chat/... - type: string - gallery: - type: array - description: >- - URLs or paths in the runcitadel/app-images/[app-name] folder with app - images - items: - type: string - path: - description: The path of the app's visible site the open button should open - type: string - defaultPassword: - description: The app's default password - type: string - torOnly: - description: Whether the app is only available over tor - type: boolean - mainContainer: - type: string - description: >- - The name of the main container for the app. If set, IP, port, and - hidden service will be assigned to it automatically. - updateContainer: - type: string - description: The container the developer system should automatically update. - required: - - name - - version - - category - - tagline - - description - - developer - - website - - repo - - support - - gallery - additionalProperties: false - - containers: - type: array - items: - type: object - properties: - name: - type: string - image: - type: string - permissions: - type: array - items: - type: string - enum: - - lnd - - bitcoind - - electrum - - root - - hw - ports: - type: array - items: - type: - - string - - number - port: - type: number - description: >- - If this is the main container, the port inside the container which - will be exposed to the outside as the port specified in metadata. - environment: - type: object - data: - type: array - description: >- - An array of at directories in the container the app stores its data - in. Can be empty. Please only list top-level directories. - items: - type: string - user: - type: string - description: The user the container should run as - stop_grace_period: - type: string - description: The grace period for stopping the container. Defaults to 1 minute. - depends_on: - type: array - description: The services the container depends on - entrypoint: - type: - - string - - array - description: The entrypoint for the container - bitcoin_mount_dir: - type: string - description: Where to mount the bitcoin dir - command: - type: - - string - - array - description: The command for the container - init: - type: boolean - description: Whether the container should be run with init - stop_signal: - type: string - description: The signal to send to the container when stopping - noNetwork: - type: boolean - description: >- - Set this to true if the container shouldn't get an IP & port - exposed. - needsHiddenService: - type: boolean - description: >- - Set this to true if the container should be assigned a hidden - service even if it's not the main container. - hiddenServicePort: - type: number - description: >- - Set this to a port if your container exposes multiple ports, but - only one should be a hidden service. - hiddenServicePorts: - type: object - description: >- - Set this to a map of service names to hidden service ports if your - container exposes multiple ports, and all of them should be hidden - services. - patternProperties: - ^[a-zA-Z0-9_]+$: - type: - - number - - array - restart: - type: string - description: When the container should restart. Can be 'always' or 'on-failure'. - additionalProperties: false - required: - - name - - image - additionalProperties: false - -required: - - metadata - - containers - -additionalProperties: false diff --git a/app/lib/composegenerator/shared/env.py b/app/lib/composegenerator/shared/env.py index 69b4318..fb12f1a 100644 --- a/app/lib/composegenerator/shared/env.py +++ b/app/lib/composegenerator/shared/env.py @@ -4,7 +4,7 @@ import re from typing import Union -from lib.composegenerator.v1.types import App +from lib.composegenerator.v2.types import App from lib.composegenerator.shared.const import always_allowed_env from lib.citadelutils import checkArrayContainsAllElements, getEnvVars diff --git a/app/lib/composegenerator/shared/main.py b/app/lib/composegenerator/shared/main.py index 4cc14df..da3c553 100644 --- a/app/lib/composegenerator/shared/main.py +++ b/app/lib/composegenerator/shared/main.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # Main functions -from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container +from lib.composegenerator.v2.types import App, AppStage3, AppStage2, Container from lib.composegenerator.shared.const import permissions diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py new file mode 100644 index 0000000..0b4e47e --- /dev/null +++ b/app/lib/composegenerator/shared/networking.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +import json +from os import path +import random +from lib.composegenerator.v2.types import ContainerStage2, NetworkConfig +from lib.citadelutils import parse_dotenv +from dacite import from_dict + +def getFreePort(networkingFile: str, appId: str): + # Ports used currently in Citadel + usedPorts = [ + # Dashboard + 80, + # Sometimes used by nginx with some setups + 433, + # Dashboard SSL + 443, + # Bitcoin Core P2P + 8333, + # LND gRPC + 10009, + # LND REST + 8080, + # Electrum Server + 50001, + # Tor Proxy + 9050, + ] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + if 'ports' in networkingData: + usedPorts += list(networkingData['ports'].values()) + else: + networkingData['ports'] = {} + + if appId in networkingData['ports']: + return networkingData['ports'][appId] + + while True: + port = str(random.randint(1024, 49151)) + if port not in usedPorts: + # Check if anyhing is listening on the specific port + if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: + networkingData['ports'][appId] = port + break + + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + + return port + +def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1: + raise Exception("Newline in container name") + env_var = "APP_{}_{}_IP".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP + # can be used + usedIps = [] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + + if 'ip_addresses' in networkingData: + usedIps = list(networkingData['ip_addresses'].values()) + else: + networkingData['ip_addresses'] = {} + # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container + # If the IP is already in use, it will be tried again until it's not in use + # If it's not in use, it will be added to the usedIps list and written to the usedIpFile + # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), + # Throw an error, because no more IPs can be used + if len(usedIps) == 235: + raise Exception("No more IPs can be used") + + if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: + ip = networkingData['ip_addresses']["{}-{}".format( + appId, container.name)] + else: + while True: + ip = "10.21.21." + str(random.randint(20, 255)) + if ip not in usedIps: + networkingData['ip_addresses']["{}-{}".format( + appId, container.name)] = ip + break + container.networks = from_dict(data_class=NetworkConfig, data={'default': { + 'ipv4_address': "$" + env_var}}) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): + return container + + # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, ip)) + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + return container + + +def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1 or container.name.find(" ") != -1: + raise Exception("Newline or space in container name") + + env_var = "APP_{}_{}_PORT".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + + port = getFreePort(networkingFile, appId) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(port): + return {"port": port, "env_var": "${{{}}}".format(env_var)} + + # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, port)) + + # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} + # where the outer {{ }} will be replaced by {} in the returned string + return {"port": port, "env_var": "${{{}}}".format(env_var)} + diff --git a/app/lib/composegenerator/v1/generate.py b/app/lib/composegenerator/v1/generate.py deleted file mode 100644 index fa01fd4..0000000 --- a/app/lib/composegenerator/v1/generate.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from lib.composegenerator.v1.types import App, AppStage4, generateApp -from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices -from lib.composegenerator.shared.env import validateEnv -from lib.citadelutils import classToDict -import os - -def createComposeConfigFromV1(app: dict, nodeRoot: str): - envFile = os.path.join(nodeRoot, ".env") - networkingFile = os.path.join(nodeRoot, "apps", "networking.json") - - newApp: App = generateApp(app) - newApp = convertContainerPermissions(newApp) - validateEnv(newApp) - newApp = convertDataDirToVolume(newApp) - newApp = configureIps(newApp, networkingFile, envFile) - newApp = configureMainPort(newApp, nodeRoot) - configureHiddenServices(newApp, nodeRoot) - finalConfig: AppStage4 = convertContainersToServices(newApp) - newApp = classToDict(finalConfig) - del newApp['metadata'] - if "version" in newApp: - del newApp["version"] - # Set version to 3.8 (current compose file version) - newApp = {'version': '3.8', **newApp} - return newApp diff --git a/app/lib/composegenerator/v1/networking.py b/app/lib/composegenerator/v1/networking.py deleted file mode 100644 index 68d1dc1..0000000 --- a/app/lib/composegenerator/v1/networking.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from dacite import from_dict -from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig -from lib.citadelutils import parse_dotenv -import json -from os import path -import random -from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService - - -def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1: - raise Exception("Newline in container name") - env_var = "APP_{}_{}_IP".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP - # can be used - usedIps = [] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - - if 'ip_addresses' in networkingData: - usedIps = list(networkingData['ip_addresses'].values()) - else: - networkingData['ip_addresses'] = {} - # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container - # If the IP is already in use, it will be tried again until it's not in use - # If it's not in use, it will be added to the usedIps list and written to the usedIpFile - # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), - # Throw an error, because no more IPs can be used - if len(usedIps) == 235: - raise Exception("No more IPs can be used") - - if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: - ip = networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] - else: - while True: - ip = "10.21.21." + str(random.randint(20, 255)) - if ip not in usedIps: - networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] = ip - break - container.networks = from_dict(data_class=NetworkConfig, data={'default': { - 'ipv4_address': "$" + env_var}}) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): - return container - - # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, ip)) - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - return container - - -def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1 or container.name.find(" ") != -1: - raise Exception("Newline or space in container name") - - env_var = "APP_{}_{}_PORT".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - - port = getFreePort(networkingFile, appId) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(port): - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, port)) - - # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} - # where the outer {{ }} will be replaced by {} in the returned string - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - -def getMainContainer(app: dict): - if len(app.containers) == 1: - return app.containers[0] - else: - if not app.metadata.mainContainer: - app.metadata.mainContainer = 'main' - for container in app.containers: - if container.name == app.metadata.mainContainer: - return container - raise Exception( - "No main container found for app {}".format(app.metadata.name)) - - -def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: - registryFile = path.join(nodeRoot, "apps", "registry.json") - registry: list = [] - if path.isfile(registryFile): - with open(registryFile, 'r') as f: - registry = json.load(f) - else: - raise Exception("Registry file not found") - - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - - mainContainer = getMainContainer(app) - - portDetails = assignPort(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - containerPort = portDetails['port'] - portAsEnvVar = portDetails['env_var'] - portToAppend = portAsEnvVar - - mainPort = False - - if mainContainer.port: - portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port) - mainPort = mainContainer.port - del mainContainer.port - else: - portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar) - - if mainContainer.ports: - mainContainer.ports.append(portToAppend) - # Set the main port to the first port in the list, if it contains a :, it's the port after the : - # If it doesn't contain a :, it's the port itself - if mainPort == False: - mainPort = mainContainer.ports[0] - if mainPort.find(":") != -1: - mainPort = mainPort.split(":")[1] - else: - mainContainer.ports = [portToAppend] - if mainPort == False: - mainPort = portDetails['port'] - - mainContainer = assignIp(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - - # If the IP wasn't in dotenv before, now it should be - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - - containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace( - "-", "_"), mainContainer.name.upper().replace("-", "_"))] - - hiddenservice = getHiddenService( - app.metadata.name, app.metadata.id, containerIP, mainPort) - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenservice) - - # Also set the port in metadata - app.metadata.port = int(containerPort) - - for registryApp in registry: - if registryApp['id'] == app.metadata.id: - registry[registry.index(registryApp)]['port'] = int(containerPort) - break - - with open(registryFile, 'w') as f: - json.dump(registry, f, indent=4, sort_keys=True) - - return app - - -def configureIps(app: AppStage2, networkingFile: str, envFile: str): - for container in app.containers: - if container.noNetwork: - # Check if port is defined for the container - if container.port: - raise Exception("Port defined for container without network") - if app.metadata.mainContainer == container.name: - raise Exception("Main container without network") - # Skip this iteration of the loop - continue - - container = assignIp(container, app.metadata.id, - networkingFile, envFile) - - return app - - -def configureHiddenServices(app: dict, nodeRoot: str) -> None: - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - hiddenServices = "" - - if len(app.containers) == 1: - mainContainer = app.containers[0] - else: - mainContainer = None - if app.metadata.mainContainer == None: - app.metadata.mainContainer = 'main' - for container in app.containers: - if container.name == app.metadata.mainContainer: - mainContainer = container - break - if mainContainer is None: - raise Exception("No main container found") - - for container in app.containers: - env_var = "APP_{}_{}_IP".format( - app.metadata.id.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - hiddenServices += getContainerHiddenService( - app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name) - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenServices) diff --git a/app/lib/composegenerator/v1/types.py b/app/lib/composegenerator/v1/types.py deleted file mode 100644 index c9556be..0000000 --- a/app/lib/composegenerator/v1/types.py +++ /dev/null @@ -1,151 +0,0 @@ -from typing import Union, List -from dataclasses import dataclass, field -from dacite import from_dict - -@dataclass -class Metadata: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - repo: str - support: str - gallery: List[str] = field(default_factory=list) - dependencies: List[str] = field(default_factory=list) - mainContainer: Union[str, None] = None - updateContainer: Union[str, None] = None - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - -@dataclass -class Container: - name: str - image: str - permissions: list = field(default_factory=list) - ports: list = field(default_factory=list) - port: Union[int, None] = None - environment: Union[dict, None] = None - data: list = field(default_factory=list) - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: list = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - bitcoin_mount_dir: Union[str, None] = None - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - environment_allow: list = field(default_factory=list) - # Only added later - volumes: list = field(default_factory=list) - restart: Union[str, None] = None - -@dataclass -class App: - version: Union[str, int] - metadata: Metadata - containers: List[Container] - -# Generate an app instance from an app dict -def generateApp(appDict): - return from_dict(data_class=App, data=appDict) - -@dataclass -class Network: - ipv4_address: Union[str, None] = None - -@dataclass -class NetworkConfig: - default: Network - -# After converting data dir and defining volumes, stage 2 -@dataclass -class ContainerStage2: - id: str - name: str - image: str - permissions: List[str] = field(default_factory=list) - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - -@dataclass -class AppStage2: - version: Union[str, int] - metadata: Metadata - containers: List[ContainerStage2] - -@dataclass -class MetadataStage3: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - dependencies: List[str] - repo: str - support: str - gallery: List[str] - mainContainer: Union[str, None] = None - updateContainer: Union[str, None] = None - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - -@dataclass -class AppStage3: - version: Union[str, int] - metadata: MetadataStage3 - containers: List[ContainerStage2] - -@dataclass -class ContainerStage4: - id: str - name: str - image: str - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - -@dataclass -class AppStage4: - version: Union[str, int] - metadata: MetadataStage3 - services: List[ContainerStage4] \ No newline at end of file diff --git a/app/lib/composegenerator/v1/utils/networking.py b/app/lib/composegenerator/v1/utils/networking.py deleted file mode 100644 index 2668aa7..0000000 --- a/app/lib/composegenerator/v1/utils/networking.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import json -import os -import random - -from lib.composegenerator.v1.types import Container - -def getFreePort(networkingFile: str, appId: str): - # Ports used currently in Citadel - usedPorts = [ - # Dashboard - 80, - # Sometimes used by nginx with some setups - 433, - # Dashboard SSL - 443, - # Bitcoin Core P2P - 8333, - # LND gRPC - 10009, - # LND REST - 8080, - # Electrum Server - 50001, - # Tor Proxy - 9050, - ] - networkingData = {} - if os.path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - if 'ports' in networkingData: - usedPorts += list(networkingData['ports'].values()) - else: - networkingData['ports'] = {} - - if appId in networkingData['ports']: - return networkingData['ports'][appId] - - while True: - port = str(random.randint(1024, 49151)) - if port not in usedPorts: - # Check if anyhing is listening on the specific port - if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: - networkingData['ports'][appId] = port - break - - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - - return port - - -def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str: - hiddenServices = ''' -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -'''.format(name, id) - for port in ports: - hiddenServices += 'HiddenServicePort {} {}:{}'.format( - port, internalIp, port) - hiddenServices += "\n" - return hiddenServices - - -def getHiddenServiceString(name: str, id: str, internalPort, internalIp: str, publicPort) -> str: - return ''' -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -HiddenServicePort {} {}:{} - -'''.format(name, id, publicPort, internalIp, internalPort) - - -def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str: - return getHiddenServiceString(appName, appId, appPort, appIp, "80") - - -def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str: - if not container.needsHiddenService and not isMainContainer: - return "" - if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer: - print("Container {} for app {} isn't compatible with hidden service assignment".format( - container.name, appName)) - return "" - - if isMainContainer: - if not container.hiddenServicePorts: - return "" - # hiddenServicePorts is a map of hidden service name to port - # We need to generate a hidden service for each one - hiddenServices = "" - for name, port in container.hiddenServicePorts.items(): - if ".." in name: - print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName)) - - # If port is a list, use getHiddenServiceMultiPort - if isinstance(port, list): - hiddenServices += getHiddenServiceMultiPort("{} {}".format(appName, name), "{}-{}".format( - appId, name), containerIp, port) - else: - hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format( - appId, name), port, containerIp, port) - del container.hiddenServicePorts - return hiddenServices - - del container.needsHiddenService - if not container.port: - data = getHiddenServiceString(appName + container.name, "{}-{}".format( - appId, container.name), container.hiddenServicePort, containerIp, "80") - del container.hiddenServicePort - return data - else: - return getHiddenServiceString(appName + container.name, "{}-{}".format( - appId, container.name), container.port, containerIp, container.port) diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py index ec5d4ad..a69540d 100644 --- a/app/lib/composegenerator/v2/networking.py +++ b/app/lib/composegenerator/v2/networking.py @@ -8,7 +8,7 @@ import json from os import path import random from lib.composegenerator.v2.utils.networking import getContainerHiddenService -from lib.composegenerator.v1.networking import assignIp, assignPort +from lib.composegenerator.shared.networking import assignIp, assignPort def getMainContainer(app: App) -> Container: diff --git a/app/lib/composegenerator/v3/networking.py b/app/lib/composegenerator/v3/networking.py index 66d5a96..2e25f55 100644 --- a/app/lib/composegenerator/v3/networking.py +++ b/app/lib/composegenerator/v3/networking.py @@ -7,8 +7,7 @@ from lib.citadelutils import parse_dotenv import json from os import path import random -from lib.composegenerator.v1.networking import assignIp, assignPort - +from lib.composegenerator.shared.networking import assignIp, assignPort def getMainContainerIndex(app: App): if len(app.containers) == 1: @@ -105,4 +104,3 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: with open(envFile, 'a') as f: f.write("{}={}\n".format(portAsEnvVar, app.metadata.port)) return app - diff --git a/app/lib/manage.py b/app/lib/manage.py index ea41d1e..7ae9a48 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -27,7 +27,6 @@ except Exception: print("Continuing anyway, but some features won't be available,") print("for example checking for app updates") -from lib.composegenerator.v1.generate import createComposeConfigFromV1 from lib.composegenerator.v2.generate import createComposeConfigFromV2 from lib.composegenerator.v3.generate import createComposeConfigFromV3 from lib.validate import findAndValidateApps @@ -236,10 +235,7 @@ def getApp(app, appId: str): raise Exception("Error: Could not find metadata in " + appFile) app["metadata"]["id"] = appId - if 'version' in app and str(app['version']) == "1": - print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId)) - return createComposeConfigFromV1(app, nodeRoot) - elif 'version' in app and str(app['version']) == "2": + if 'version' in app and str(app['version']) == "2": print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId)) return createComposeConfigFromV2(app, nodeRoot) elif 'version' in app and str(app['version']) == "3": diff --git a/app/lib/validate.py b/app/lib/validate.py index db3d5e0..2b6ba09 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -11,8 +11,6 @@ import traceback scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") nodeRoot = os.path.join(scriptDir, "..") -with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f: - schemaVersion1 = yaml.safe_load(f) with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: schemaVersion2 = yaml.safe_load(f) with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: @@ -24,15 +22,7 @@ with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file: # Validates app data # Returns true if valid, false otherwise def validateApp(app: dict): - if 'version' in app and str(app['version']) == "1": - try: - validate(app, schemaVersion1) - return True - # Catch and log any errors, and return false - except Exception as e: - print(e) - return False - elif 'version' in app and str(app['version']) == "2": + if 'version' in app and str(app['version']) == "2": try: validate(app, schemaVersion2) return True diff --git a/docker-compose.yml b/docker-compose.yml index eef9bf3..e533d51 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -235,6 +235,13 @@ services: networks: default: ipv4_address: $REDIS_IP + + app-cli: + container_name: app-cli + image: ghcr.io/runcitadel/app-cli:main@sha256:694e52fa9da1ac976165f269c17e27803032a05a76293dfe3589a50813306ded + volumes: + - ${PWD}/apps:/apps + networks: default: name: citadel_main_network From 91999d582a866b8f407f493d097e73c9fe780942 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sat, 16 Jul 2022 19:28:39 +0200 Subject: [PATCH 22/33] Implement quick updates (#56) Co-authored-by: nolim1t - f6287b82CC84bcbd Co-authored-by: Philipp Walter --- app/lib/composegenerator/shared/networking.py | 53 +++++++++++++++++++ docker-compose.yml | 7 --- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py index 0b4e47e..ff9bc98 100644 --- a/app/lib/composegenerator/shared/networking.py +++ b/app/lib/composegenerator/shared/networking.py @@ -54,6 +54,59 @@ def getFreePort(networkingFile: str, appId: str): return port +def assignIpV4(appId: str, containerName: str): + scriptDir = path.dirname(path.realpath(__file__)) + nodeRoot = path.join(scriptDir, "..", "..", "..", "..") + networkingFile = path.join(nodeRoot, "apps", "networking.json") + envFile = path.join(nodeRoot, ".env") + cleanContainerName = containerName.strip() + # If the name still contains a newline, throw an error + if cleanContainerName.find("\n") != -1: + raise Exception("Newline in container name") + env_var = "APP_{}_{}_IP".format( + appId.upper().replace("-", "_"), + cleanContainerName.upper().replace("-", "_") + ) + # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP + # can be used + usedIps = [] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + + if 'ip_addresses' in networkingData: + usedIps = list(networkingData['ip_addresses'].values()) + else: + networkingData['ip_addresses'] = {} + # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container + # If the IP is already in use, it will be tried again until it's not in use + # If it's not in use, it will be added to the usedIps list and written to the usedIpFile + # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), + # Throw an error, because no more IPs can be used + if len(usedIps) == 235: + raise Exception("No more IPs can be used") + + if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']: + ip = networkingData['ip_addresses']["{}-{}".format( + appId, cleanContainerName)] + else: + while True: + ip = "10.21.21." + str(random.randint(20, 255)) + if ip not in usedIps: + networkingData['ip_addresses']["{}-{}".format( + appId, cleanContainerName)] = ip + break + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): + return + + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, ip)) + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: # Strip leading/trailing whitespace from container.name container.name = container.name.strip() diff --git a/docker-compose.yml b/docker-compose.yml index e533d51..fa79707 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,6 @@ services: default: ipv4_address: $LND_IP dashboard: - container_name: dashboard image: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e restart: on-failure stop_grace_period: 1m30s @@ -235,12 +234,6 @@ services: networks: default: ipv4_address: $REDIS_IP - - app-cli: - container_name: app-cli - image: ghcr.io/runcitadel/app-cli:main@sha256:694e52fa9da1ac976165f269c17e27803032a05a76293dfe3589a50813306ded - volumes: - - ${PWD}/apps:/apps networks: default: From 846553dba3a36de00c55c7a8ae584a7f567c7e62 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 10:30:48 +0200 Subject: [PATCH 23/33] Remove reference to non-existent var --- app/lib/manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index 7ae9a48..159fc38 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -232,7 +232,7 @@ def stopInstalled(): # Loads an app.yml and converts it to a docker-compose.yml def getApp(app, appId: str): if not "metadata" in app: - raise Exception("Error: Could not find metadata in " + appFile) + raise Exception("Error: Could not find metadata in " + appId) app["metadata"]["id"] = appId if 'version' in app and str(app['version']) == "2": From e381f2a002b24f70074d276f5d478bf5253b275d Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 10:31:01 +0200 Subject: [PATCH 24/33] Update compose --- db/dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/dependencies.yml b/db/dependencies.yml index dac2e22..8ddedde 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -1,4 +1,4 @@ -compose: v2.10.0 +compose: v2.10.2 dashboard: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e manager: ghcr.io/runcitadel/manager:v0.0.17@sha256:ba436a07d6f96282217851756d8c81aeaa03c42dfa2246a89a78fc3384eed3cb middleware: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae From 33b06241a98c12a4bdbfdadca8d8eef02a0bb57e Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 10:45:26 +0200 Subject: [PATCH 25/33] Better separation of composegenertor v2 and v3 --- app/lib/composegenerator/shared/main.py | 27 +--- app/lib/composegenerator/shared/networking.py | 111 +++++++--------- app/lib/composegenerator/v2/generate.py | 25 +++- app/lib/composegenerator/v2/networking.py | 119 +++++++++++------- app/lib/composegenerator/v3/generate.py | 4 +- app/lib/composegenerator/v3/networking.py | 6 +- app/lib/metadata.py | 1 - 7 files changed, 146 insertions(+), 147 deletions(-) diff --git a/app/lib/composegenerator/shared/main.py b/app/lib/composegenerator/shared/main.py index da3c553..bcab1de 100644 --- a/app/lib/composegenerator/shared/main.py +++ b/app/lib/composegenerator/shared/main.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # Main functions -from lib.composegenerator.v2.types import App, AppStage3, AppStage2, Container +from lib.composegenerator.v2.types import App, AppStage3, AppStage2 from lib.composegenerator.shared.const import permissions @@ -27,28 +27,3 @@ def convertContainersToServices(app: AppStage3) -> AppStage3: del app.containers app.services = services return app - -# Converts the data of every container in app.containers to a volume, which is then added to the app -def convertDataDirToVolume(app: App) -> AppStage2: - for container in app.containers: - # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes - # Also, a datadir shouldn't start with a / - for dataDir in container.data: - if dataDir.find("..") == -1 and dataDir[0] != "/": - container.volumes.append( - '${APP_DATA_DIR}/' + dataDir) - else: - print("Data dir " + dataDir + - " contains invalid characters") - del container.data - if container.bitcoin_mount_dir != None: - if not 'bitcoind' in container.permissions: - print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name)) - # Skip this container - continue - # Also skip the container if container.bitcoin_mount_dir contains a : - if container.bitcoin_mount_dir.find(":") == -1: - container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir) - del container.bitcoin_mount_dir - - return app diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py index ff9bc98..55422b8 100644 --- a/app/lib/composegenerator/shared/networking.py +++ b/app/lib/composegenerator/shared/networking.py @@ -5,54 +5,21 @@ import json from os import path import random -from lib.composegenerator.v2.types import ContainerStage2, NetworkConfig +from app.lib.composegenerator.v2.utils.networking import getContainerHiddenService +from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container from lib.citadelutils import parse_dotenv from dacite import from_dict -def getFreePort(networkingFile: str, appId: str): - # Ports used currently in Citadel - usedPorts = [ - # Dashboard - 80, - # Sometimes used by nginx with some setups - 433, - # Dashboard SSL - 443, - # Bitcoin Core P2P - 8333, - # LND gRPC - 10009, - # LND REST - 8080, - # Electrum Server - 50001, - # Tor Proxy - 9050, - ] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - if 'ports' in networkingData: - usedPorts += list(networkingData['ports'].values()) +def getMainContainer(app: App) -> Container: + if len(app.containers) == 1: + return app.containers[0] else: - networkingData['ports'] = {} - - if appId in networkingData['ports']: - return networkingData['ports'][appId] - - while True: - port = str(random.randint(1024, 49151)) - if port not in usedPorts: - # Check if anyhing is listening on the specific port - if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: - networkingData['ports'][appId] = port - break - - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - - return port + for container in app.containers: + # Main is recommended, support web for easier porting from Umbrel + if container.name == 'main' or container.name == 'web': + return container + # Fallback to first container + return app.containers[0] def assignIpV4(appId: str, containerName: str): scriptDir = path.dirname(path.realpath(__file__)) @@ -161,30 +128,44 @@ def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFil json.dump(networkingData, f) return container +def configureIps(app: AppStage2, networkingFile: str, envFile: str): + for container in app.containers: + if container.network_mode and container.network_mode == "host": + continue + if container.noNetwork: + # Check if port is defined for the container + if container.port: + raise Exception("Port defined for container without network") + if getMainContainer(app).name == container.name: + raise Exception("Main container without network") + # Skip this iteration of the loop + continue -def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1 or container.name.find(" ") != -1: - raise Exception("Newline or space in container name") + container = assignIp(container, app.metadata.id, + networkingFile, envFile) - env_var = "APP_{}_{}_PORT".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) + return app - port = getFreePort(networkingFile, appId) +def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3: + dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) + hiddenServices = "" - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(port): - return {"port": port, "env_var": "${{{}}}".format(env_var)} + mainContainer = getMainContainer(app) - # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, port)) - - # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} - # where the outer {{ }} will be replaced by {} in the returned string - return {"port": port, "env_var": "${{{}}}".format(env_var)} + for container in app.containers: + if container.network_mode and container.network_mode == "host": + continue + env_var = "APP_{}_{}_IP".format( + app.metadata.id.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + hiddenServices += getContainerHiddenService( + app.metadata, container, dotEnv[env_var], container.name == mainContainer.name) + if container.hiddenServicePorts: + del container.hiddenServicePorts + torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] + torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] + with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: + f.write(hiddenServices) + return app diff --git a/app/lib/composegenerator/v2/generate.py b/app/lib/composegenerator/v2/generate.py index dfcd38d..fbb08bd 100644 --- a/app/lib/composegenerator/v2/generate.py +++ b/app/lib/composegenerator/v2/generate.py @@ -3,15 +3,34 @@ # SPDX-License-Identifier: GPL-3.0-or-later from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp -from lib.composegenerator.v2.networking import configureHiddenServices, configureIps, configureMainPort -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices +from lib.composegenerator.v2.networking import configureMainPort +from lib.composegenerator.shared.networking import configureHiddenServices, configureIps +from lib.composegenerator.shared.main import convertContainerPermissions, convertContainersToServices from lib.composegenerator.shared.env import validateEnv from lib.citadelutils import classToDict import os def convertDataDirToVolumeGen2(app: App) -> AppStage2: - app = convertDataDirToVolume(app) for container in app.containers: + # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes + # Also, a datadir shouldn't start with a / + for dataDir in container.data: + if dataDir.find("..") == -1 and dataDir[0] != "/": + container.volumes.append( + '${APP_DATA_DIR}/' + dataDir) + else: + print("Data dir " + dataDir + + " contains invalid characters") + del container.data + if container.bitcoin_mount_dir != None: + if not 'bitcoind' in container.permissions: + print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name)) + # Skip this container + continue + # Also skip the container if container.bitcoin_mount_dir contains a : + if container.bitcoin_mount_dir.find(":") == -1: + container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir) + del container.bitcoin_mount_dir if container.lnd_mount_dir != None: if not 'lnd' in container.permissions: print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name)) diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py index a69540d..eb01041 100644 --- a/app/lib/composegenerator/v2/networking.py +++ b/app/lib/composegenerator/v2/networking.py @@ -2,14 +2,58 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +from app.lib.citadelutils import parse_dotenv from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container -from lib.citadelutils import parse_dotenv import json from os import path +import os import random -from lib.composegenerator.v2.utils.networking import getContainerHiddenService -from lib.composegenerator.shared.networking import assignIp, assignPort +from lib.composegenerator.shared.networking import assignIp +def getFreePort(networkingFile: str, appId: str): + # Ports used currently in Citadel + usedPorts = [ + # Dashboard + 80, + # Sometimes used by nginx with some setups + 433, + # Dashboard SSL + 443, + # Bitcoin Core P2P + 8333, + # LND gRPC + 10009, + # LND REST + 8080, + # Electrum Server + 50001, + # Tor Proxy + 9050, + ] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + if 'ports' in networkingData: + usedPorts += list(networkingData['ports'].values()) + else: + networkingData['ports'] = {} + + if appId in networkingData['ports']: + return networkingData['ports'][appId] + + while True: + port = str(random.randint(1024, 49151)) + if port not in usedPorts: + # Check if anyhing is listening on the specific port + if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: + networkingData['ports'][appId] = port + break + + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + + return port def getMainContainer(app: App) -> Container: if len(app.containers) == 1: @@ -22,6 +66,32 @@ def getMainContainer(app: App) -> Container: # Fallback to first container return app.containers[0] +def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1 or container.name.find(" ") != -1: + raise Exception("Newline or space in container name") + + env_var = "APP_{}_{}_PORT".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + + port = getFreePort(networkingFile, appId) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(port): + return {"port": port, "env_var": "${{{}}}".format(env_var)} + + # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, port)) + + # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} + # where the outer {{ }} will be replaced by {} in the returned string + return {"port": port, "env_var": "${{{}}}".format(env_var)} + def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: registryFile = path.join(nodeRoot, "apps", "registry.json") @@ -83,46 +153,3 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: json.dump(registry, f, indent=4, sort_keys=True) return app - -def configureIps(app: AppStage2, networkingFile: str, envFile: str): - for container in app.containers: - if container.network_mode and container.network_mode == "host": - continue - if container.noNetwork: - # Check if port is defined for the container - if container.port: - raise Exception("Port defined for container without network") - if getMainContainer(app).name == container.name: - raise Exception("Main container without network") - # Skip this iteration of the loop - continue - - container = assignIp(container, app.metadata.id, - networkingFile, envFile) - - return app - - -def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3: - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - hiddenServices = "" - - mainContainer = getMainContainer(app) - - for container in app.containers: - if container.network_mode and container.network_mode == "host": - continue - env_var = "APP_{}_{}_IP".format( - app.metadata.id.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - hiddenServices += getContainerHiddenService( - app.metadata, container, dotEnv[env_var], container.name == mainContainer.name) - if container.hiddenServicePorts: - del container.hiddenServicePorts - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenServices) - return app diff --git a/app/lib/composegenerator/v3/generate.py b/app/lib/composegenerator/v3/generate.py index 3e2789f..1164bb1 100644 --- a/app/lib/composegenerator/v3/generate.py +++ b/app/lib/composegenerator/v3/generate.py @@ -5,9 +5,9 @@ import os from lib.citadelutils import classToDict -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainersToServices +from lib.composegenerator.shared.main import convertContainersToServices from lib.composegenerator.shared.env import validateEnv -from lib.composegenerator.v2.networking import configureIps, configureHiddenServices +from lib.composegenerator.shared.networking import configureIps, configureHiddenServices from lib.composegenerator.v3.types import App, AppStage2, AppStage4, generateApp from lib.composegenerator.v3.networking import configureMainPort diff --git a/app/lib/composegenerator/v3/networking.py b/app/lib/composegenerator/v3/networking.py index 2e25f55..ac784d0 100644 --- a/app/lib/composegenerator/v3/networking.py +++ b/app/lib/composegenerator/v3/networking.py @@ -2,12 +2,10 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from lib.composegenerator.v3.types import App, AppStage2, AppStage3, Container -from lib.citadelutils import parse_dotenv +from lib.composegenerator.v3.types import App, AppStage2, AppStage3 import json from os import path -import random -from lib.composegenerator.shared.networking import assignIp, assignPort +from lib.composegenerator.shared.networking import assignIp def getMainContainerIndex(app: App): if len(app.containers) == 1: diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 69b0f1e..246ab62 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -6,7 +6,6 @@ import os import yaml import traceback -from lib.composegenerator.v2.networking import getMainContainer from lib.composegenerator.shared.networking import assignIpV4 from lib.entropy import deriveEntropy from typing import List From 066f2c7f9ef5048a38c4d4b247e41b85cf0c0f27 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 10:46:51 +0200 Subject: [PATCH 26/33] Mark 0.1.5 as removal version for app.yml v2 and v3 --- app/lib/manage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index 159fc38..7678a44 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -236,10 +236,10 @@ def getApp(app, appId: str): app["metadata"]["id"] = appId if 'version' in app and str(app['version']) == "2": - print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId)) + print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId)) return createComposeConfigFromV2(app, nodeRoot) elif 'version' in app and str(app['version']) == "3": - print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.3.0".format(appId)) + print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId)) return createComposeConfigFromV3(app, nodeRoot) else: raise Exception("Error: Unsupported version of app.yml") From ac4cac92a95793dc5b092ed4bf94f30eec97b015 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 10:48:40 +0200 Subject: [PATCH 27/33] Fix imports --- app/lib/composegenerator/shared/networking.py | 2 +- app/lib/composegenerator/v2/networking.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py index 55422b8..8208fd5 100644 --- a/app/lib/composegenerator/shared/networking.py +++ b/app/lib/composegenerator/shared/networking.py @@ -5,7 +5,7 @@ import json from os import path import random -from app.lib.composegenerator.v2.utils.networking import getContainerHiddenService +from lib.composegenerator.v2.utils.networking import getContainerHiddenService from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container from lib.citadelutils import parse_dotenv from dacite import from_dict diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py index eb01041..fd9ec5c 100644 --- a/app/lib/composegenerator/v2/networking.py +++ b/app/lib/composegenerator/v2/networking.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from app.lib.citadelutils import parse_dotenv +from lib.citadelutils import parse_dotenv from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container import json from os import path From 34c97f8b33031c765c8b5b4cb0726692e1370c6c Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 11:42:05 +0200 Subject: [PATCH 28/33] Remove another unused import --- app/lib/composegenerator/shared/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/composegenerator/shared/main.py b/app/lib/composegenerator/shared/main.py index bcab1de..ff9922d 100644 --- a/app/lib/composegenerator/shared/main.py +++ b/app/lib/composegenerator/shared/main.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # Main functions -from lib.composegenerator.v2.types import App, AppStage3, AppStage2 +from lib.composegenerator.v2.types import App, AppStage3 from lib.composegenerator.shared.const import permissions From fd3a31c8b5b3e8e877174a5512adffe2d4cc4e39 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 11:42:58 +0200 Subject: [PATCH 29/33] Move utils to shared --- app/lib/composegenerator/shared/networking.py | 2 +- app/lib/composegenerator/{v2 => shared}/utils/networking.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename app/lib/composegenerator/{v2 => shared}/utils/networking.py (100%) diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py index 8208fd5..7c1c397 100644 --- a/app/lib/composegenerator/shared/networking.py +++ b/app/lib/composegenerator/shared/networking.py @@ -5,7 +5,7 @@ import json from os import path import random -from lib.composegenerator.v2.utils.networking import getContainerHiddenService +from lib.composegenerator.shared.utils.networking import getContainerHiddenService from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container from lib.citadelutils import parse_dotenv from dacite import from_dict diff --git a/app/lib/composegenerator/v2/utils/networking.py b/app/lib/composegenerator/shared/utils/networking.py similarity index 100% rename from app/lib/composegenerator/v2/utils/networking.py rename to app/lib/composegenerator/shared/utils/networking.py From af847c01012a753af3e2da90345f9d3b05a37b04 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 12:21:16 +0200 Subject: [PATCH 30/33] Fix app port reassignment & race condition when writing to registry.json --- app/lib/citadelutils.py | 26 ++++++++++++++++ app/lib/composegenerator/v2/networking.py | 5 ++- app/lib/composegenerator/v3/networking.py | 4 +++ app/lib/manage.py | 37 ++++++----------------- app/lib/metadata.py | 25 ++++++++------- 5 files changed, 58 insertions(+), 39 deletions(-) diff --git a/app/lib/citadelutils.py b/app/lib/citadelutils.py index e0b044e..bf8d893 100644 --- a/app/lib/citadelutils.py +++ b/app/lib/citadelutils.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: GPL-3.0-or-later import re +import fcntl +import os # Helper functions @@ -77,3 +79,27 @@ def classToDict(theClass): obj[key] = classToDict(value) return obj +class FileLock: + """Implements a file-based lock using flock(2). + The lock file is saved in directory dir with name lock_name. + dir is the current directory by default. + """ + + def __init__(self, lock_name, dir="."): + self.lock_file = open(os.path.join(dir, lock_name), "w") + + def acquire(self, blocking=True): + """Acquire the lock. + If the lock is not already acquired, return None. If the lock is + acquired and blocking is True, block until the lock is released. If + the lock is acquired and blocking is False, raise an IOError. + """ + ops = fcntl.LOCK_EX + if not blocking: + ops |= fcntl.LOCK_NB + fcntl.flock(self.lock_file, ops) + + def release(self): + """Release the lock. Return None even if lock not currently acquired""" + fcntl.flock(self.lock_file, fcntl.LOCK_UN) + \ No newline at end of file diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py index fd9ec5c..e144867 100644 --- a/app/lib/composegenerator/v2/networking.py +++ b/app/lib/composegenerator/v2/networking.py @@ -9,6 +9,7 @@ from os import path import os import random from lib.composegenerator.shared.networking import assignIp +from lib.citadelutils import FileLock def getFreePort(networkingFile: str, appId: str): # Ports used currently in Citadel @@ -94,6 +95,8 @@ def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: + lock = FileLock("citadel_registry_lock", dir="/tmp") + lock.acquire() registryFile = path.join(nodeRoot, "apps", "registry.json") registry: list = [] if path.isfile(registryFile): @@ -151,5 +154,5 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: with open(registryFile, 'w') as f: json.dump(registry, f, indent=4, sort_keys=True) - + lock.release() return app diff --git a/app/lib/composegenerator/v3/networking.py b/app/lib/composegenerator/v3/networking.py index ac784d0..edb696c 100644 --- a/app/lib/composegenerator/v3/networking.py +++ b/app/lib/composegenerator/v3/networking.py @@ -6,6 +6,7 @@ from lib.composegenerator.v3.types import App, AppStage2, AppStage3 import json from os import path from lib.composegenerator.shared.networking import assignIp +from lib.citadelutils import FileLock def getMainContainerIndex(app: App): if len(app.containers) == 1: @@ -24,6 +25,8 @@ def getMainContainerIndex(app: App): def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: + lock = FileLock("citadel_registry_lock", dir="/tmp") + lock.acquire() registryFile = path.join(nodeRoot, "apps", "registry.json") portsFile = path.join(nodeRoot, "apps", "ports.json") envFile = path.join(nodeRoot, ".env") @@ -98,6 +101,7 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: with open(registryFile, 'w') as f: json.dump(registry, f, indent=4, sort_keys=True) + lock.release() with open(envFile, 'a') as f: f.write("{}={}\n".format(portAsEnvVar, app.metadata.port)) diff --git a/app/lib/manage.py b/app/lib/manage.py index 7678a44..d3eb644 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -10,7 +10,6 @@ import random from typing import List from sys import argv import os -import fcntl import requests import shutil import json @@ -32,30 +31,7 @@ from lib.composegenerator.v3.generate import createComposeConfigFromV3 from lib.validate import findAndValidateApps from lib.metadata import getAppRegistry from lib.entropy import deriveEntropy - -class FileLock: - """Implements a file-based lock using flock(2). - The lock file is saved in directory dir with name lock_name. - dir is the current directory by default. - """ - - def __init__(self, lock_name, dir="."): - self.lock_file = open(os.path.join(dir, lock_name), "w") - - def acquire(self, blocking=True): - """Acquire the lock. - If the lock is not already acquired, return None. If the lock is - acquired and blocking is True, block until the lock is released. If - the lock is acquired and blocking is False, raise an IOError. - """ - ops = fcntl.LOCK_EX - if not blocking: - ops |= fcntl.LOCK_NB - fcntl.flock(self.lock_file, ops) - - def release(self): - """Release the lock. Return None even if lock not currently acquired""" - fcntl.flock(self.lock_file, fcntl.LOCK_UN) +from lib.citadelutils import FileLock # For an array of threads, join them and wait for them to finish def joinThreads(threads: List[threading.Thread]): @@ -109,7 +85,7 @@ def handleAppV4(app): for registryApp in registry: if registryApp['id'] == app: - registry[registry.index(registryApp)]['port'] = resultYml["port"] + registry[registry.index(registryApp)]['port'] = mainPort break with open(registryFile, 'w') as f: @@ -136,12 +112,19 @@ def getAppYml(name): def update(verbose: bool = False): apps = findAndValidateApps(appsDir) + portCache = {} + try: + with open(os.path.join(appsDir, "ports.cache.json"), "w") as f: + portCache = json.load(f) + except Exception: pass # The compose generation process updates the registry, so we need to get it set up with the basics before that - registry = getAppRegistry(apps, appsDir) + registry = getAppRegistry(apps, appsDir, portCache) with open(os.path.join(appsDir, "registry.json"), "w") as f: json.dump(registry["metadata"], f, sort_keys=True) with open(os.path.join(appsDir, "ports.json"), "w") as f: json.dump(registry["ports"], f, sort_keys=True) + with open(os.path.join(appsDir, "ports.cache.json"), "w") as f: + json.dump(registry["portCache"], f, sort_keys=True) with open(os.path.join(appsDir, "virtual-apps.json"), "w") as f: json.dump(registry["virtual_apps"], f, sort_keys=True) print("Wrote registry to registry.json") diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 246ab62..d8679c8 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -8,9 +8,6 @@ import traceback from lib.composegenerator.shared.networking import assignIpV4 from lib.entropy import deriveEntropy -from typing import List -import json -import random appPorts = {} appPortMap = {} @@ -36,9 +33,10 @@ def appPortsToMap(): # Also check the path and defaultPassword and set them to an empty string if they don't exist # In addition, set id on the metadata to the name of the app # Return a list of all app's metadata -def getAppRegistry(apps, app_path): +def getAppRegistry(apps, app_path, portCache): app_metadata = [] virtual_apps = {} + appPorts = portCache for app in apps: app_yml_path = os.path.join(app_path, app, 'app.yml') if os.path.isfile(app_yml_path): @@ -77,7 +75,8 @@ def getAppRegistry(apps, app_path): return { "virtual_apps": virtual_apps, "metadata": app_metadata, - "ports": appPortMap + "ports": appPortMap, + "portCache": appPorts, } citadelPorts = [ @@ -101,9 +100,11 @@ citadelPorts = [ lastPort = 3000 -def getNewPort(usedPorts): +def getNewPort(usedPorts, appId, containerName, allowExisting): lastPort2 = lastPort - while lastPort2 in usedPorts or lastPort2 in citadelPorts: + while lastPort2 in usedPorts.keys() or lastPort2 in citadelPorts: + if allowExisting and lastPort2 in usedPorts.keys() and usedPorts[lastPort2]["app"] == appId and usedPorts[lastPort2]["container"] == containerName: + break lastPort2 = lastPort2 + 1 return lastPort2 @@ -117,10 +118,10 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna "dynamic": isDynamic, } else: - if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != appContainer["name"]: - newPort = getNewPort(appPorts.keys()) + if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != containerName: if port in appPorts and priority > appPorts[port]["priority"]: #print("Prioritizing app {} over {}".format(appId, appPorts[port]["app"])) + newPort = getNewPort(appPorts, appPorts[port]["app"], appPorts[port]["container"], False) appPorts[newPort] = appPorts[port].copy() appPorts[port] = { "app": appId, @@ -134,7 +135,8 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna disabledApps.append(appId) print("App {} disabled because of port conflict".format(appId)) else: - #print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, appContainer["name"], newPort, port)) + newPort = getNewPort(appPorts, appId, containerName, True) + #print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, containerName, newPort, port)) appPorts[newPort] = { "app": appId, "port": port, @@ -164,7 +166,8 @@ def getPortsV3App(app, appId): else: validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0) elif "requiredPorts" not in appContainer and "requiredUdpPorts" not in appContainer: - validatePort(appContainer["name"], appContainer, getNewPort(appPorts.keys()), appId, 0, True) + # if the container does not define a port, assume 3000, and pass it to the container as env var + validatePort(appContainer["name"], appContainer, 3000, appId, 0, True) if "requiredPorts" in appContainer: for port in appContainer["requiredPorts"]: validatePort(appContainer["name"], appContainer, port, appId, 2) From b95d1792cb39d353fedddac54121fd1de51319d7 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 13:56:40 +0200 Subject: [PATCH 31/33] Fix command --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2a6bfd2..6ed0679 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,10 +112,7 @@ services: depends_on: - tor - redis - command: - - ./start.sh restart: on-failure - init: true stop_grace_period: 5m30s volumes: - ${PWD}/info.json:/info.json From e2b27dc303d602611fcb2e3241dbd173ed71b4cc Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 16:13:59 +0200 Subject: [PATCH 32/33] Update manager --- db/dependencies.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/dependencies.yml b/db/dependencies.yml index 218cabb..9d51c8d 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -1,5 +1,5 @@ compose: v2.10.2 dashboard: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e -manager: ghcr.io/runcitadel/manager:deno@sha256:38ef8474cc501d3f3e9ea63e73d1c48f848662467ffe5f7f0b9bbb44e04055cf +manager: ghcr.io/runcitadel/manager:deno@sha256:755aad8fc745ae189f89880356ce676e5146afaf0c7b250adcd9f65a0856db81 middleware: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae app-cli: ghcr.io/runcitadel/app-cli:main@sha256:6dad26faf652b930cb219a6261af8edfa84d5db4d679153700dbc1b136267bbf diff --git a/docker-compose.yml b/docker-compose.yml index 6986c56..dcae73f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,7 +107,7 @@ services: ipv4_address: $DASHBOARD_IP manager: container_name: manager - image: ghcr.io/runcitadel/manager:deno@sha256:38ef8474cc501d3f3e9ea63e73d1c48f848662467ffe5f7f0b9bbb44e04055cf + image: ghcr.io/runcitadel/manager:deno@sha256:755aad8fc745ae189f89880356ce676e5146afaf0c7b250adcd9f65a0856db81 depends_on: - tor - redis From bc90cfcb27d4e98e506bdf721345a89b5067488f Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Aug 2022 16:28:26 +0200 Subject: [PATCH 33/33] Update manager --- db/dependencies.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/dependencies.yml b/db/dependencies.yml index 9d51c8d..21808ec 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -1,5 +1,5 @@ compose: v2.10.2 dashboard: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e -manager: ghcr.io/runcitadel/manager:deno@sha256:755aad8fc745ae189f89880356ce676e5146afaf0c7b250adcd9f65a0856db81 +manager: ghcr.io/runcitadel/manager:deno@sha256:896b98ab270355d053203c8e4a31df449b447ff1ef91ef86b1b0907400db1bd6 middleware: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae app-cli: ghcr.io/runcitadel/app-cli:main@sha256:6dad26faf652b930cb219a6261af8edfa84d5db4d679153700dbc1b136267bbf diff --git a/docker-compose.yml b/docker-compose.yml index dcae73f..adf3310 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,7 +107,7 @@ services: ipv4_address: $DASHBOARD_IP manager: container_name: manager - image: ghcr.io/runcitadel/manager:deno@sha256:755aad8fc745ae189f89880356ce676e5146afaf0c7b250adcd9f65a0856db81 + image: ghcr.io/runcitadel/manager:deno@sha256:896b98ab270355d053203c8e4a31df449b447ff1ef91ef86b1b0907400db1bd6 depends_on: - tor - redis