From c7342afafbba52936163f189907cd7044844bdab Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Wed, 15 Jun 2022 07:56:17 +0000 Subject: [PATCH 01/21] Improve list-updates --- app/lib/manage.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index 98400f6..7713d6c 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -187,22 +187,29 @@ def getUserData(): userData = json.load(f) return userData -def checkUpdateAvailable(name: str) -> bool: +def checkUpdateAvailable(name: str): latestAppYml = yaml.safe_load(getAppYml(name)) with open(os.path.join(appsDir, name, "app.yml"), "r") as f: originalAppYml = yaml.safe_load(f) if not "metadata" in latestAppYml or not "version" in latestAppYml["metadata"] or not "metadata" in originalAppYml or not "version" in originalAppYml["metadata"]: print("App {} is not valid".format(name), file=sys.stderr) return False - return semver.compare(latestAppYml["metadata"]["version"], originalAppYml["metadata"]["version"]) > 0 + if semver.compare(latestAppYml["metadata"]["version"], originalAppYml["metadata"]["version"]) > 0: + return { + "updateFrom": originalAppYml["metadata"]["version"], + "updateTo": latestAppYml["metadata"]["version"] + } + else: + return False def getAvailableUpdates(): availableUpdates = [] apps = findAndValidateApps(appsDir) for app in apps: try: - if checkUpdateAvailable(app): - availableUpdates.append(app) + checkResult = checkUpdateAvailable(app) + if checkResult: + availableUpdates.append(checkResult) except Exception: print("Warning: Can't check app {} yet".format(app), file=sys.stderr) return availableUpdates From 47e589ade8c1a3ba85ecf84e46d9f1c2b386015c Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Wed, 15 Jun 2022 08:29:16 +0000 Subject: [PATCH 02/21] Also return app names in list-updates --- app/lib/manage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index 7713d6c..92ffc4a 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -203,13 +203,13 @@ def checkUpdateAvailable(name: str): return False def getAvailableUpdates(): - availableUpdates = [] + availableUpdates = {} apps = findAndValidateApps(appsDir) for app in apps: try: checkResult = checkUpdateAvailable(app) if checkResult: - availableUpdates.append(checkResult) + availableUpdates[app] = checkResult except Exception: print("Warning: Can't check app {} yet".format(app), file=sys.stderr) return availableUpdates From 4cb179b820f88c4588ff604b8e1e0e8d874f341c Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 17 Jun 2022 07:37:40 +0000 Subject: [PATCH 03/21] Remove Bitcoin and LND from dependencies.yml --- db/dependencies.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/db/dependencies.yml b/db/dependencies.yml index 9b54520..dbdd96a 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -1,6 +1,4 @@ compose: v2.6.0 -bitcoin: 22.0 -lnd: 0.14.3 dashboard: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5 manager: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce middleware: ghcr.io/runcitadel/middleware:v0.0.11@sha256:e472da8cbfa67d9a9dbf321334fe65cdf20a0f9b6d6bab33fdf07210f54e7002 From 15138abd8e7ff39c82345c9834d72c746c46260e Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 17 Jun 2022 07:39:15 +0000 Subject: [PATCH 04/21] Restart Citadel after changing update channel --- events/triggers/set-update-channel | 1 + 1 file changed, 1 insertion(+) diff --git a/events/triggers/set-update-channel b/events/triggers/set-update-channel index 38fdec5..3371cb4 100755 --- a/events/triggers/set-update-channel +++ b/events/triggers/set-update-channel @@ -7,3 +7,4 @@ CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" "${CITADEL_ROOT}/scripts/set-update-channel" "${1}" +"${CITADEL_ROOT}/scripts/start" From 2e9704bdc9064930df12fbad99b190ec7271926d Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 17 Jun 2022 11:24:36 +0000 Subject: [PATCH 05/21] Add quick-update trigger --- events/triggers/quick-update | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 events/triggers/quick-update diff --git a/events/triggers/quick-update b/events/triggers/quick-update new file mode 100755 index 0000000..999953f --- /dev/null +++ b/events/triggers/quick-update @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)" + +RELEASE=$(cat "$CITADEL_ROOT"/statuses/update-status.json | jq .updateTo -r) + +cat < "$CITADEL_ROOT"/statuses/update-status.json +{"state": "installing", "progress": 30, "description": "Starting update", "updateTo": "$RELEASE"} +EOF + +curl "https://github.com/runcitadel/core/blob/${RELEASE}/db/dependencies.yml" > "$CITADEL_ROOT"/db/dependencies +cat < "$CITADEL_ROOT"/statuses/update-status.json +{"state": "installing", "progress": 70, "description": "Starting new containers", "updateTo": "$RELEASE"} +EOF + +"${CITADEL_ROOT}/scripts/start" + +cat < "$CITADEL_ROOT"/statuses/update-status.json +{"state": "success", "progress": 100, "description": "Successfully installed Citadel $RELEASE", "updateTo": ""} +EOF + From a000a51cac37ca2261c7e37c02165e726214c8c0 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 17 Jun 2022 11:26:52 +0000 Subject: [PATCH 06/21] Removal plan for app.yml v3 --- app/lib/manage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/lib/manage.py b/app/lib/manage.py index 92ffc4a..e0c7a3f 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -263,6 +263,7 @@ def getApp(app, appId: str): print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId)) return createComposeConfigFromV2(app, nodeRoot) elif 'version' in app and str(app['version']) == "3": + print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.3.0".format(appId)) return createComposeConfigFromV3(app, nodeRoot) else: raise Exception("Error: Unsupported version of app.yml") From 5523774b1f5d2b6853681881871245a4d430282f Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 17 Jun 2022 11:27:38 +0000 Subject: [PATCH 07/21] Fix removal message --- app/lib/manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index e0c7a3f..a4cd44d 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -263,7 +263,7 @@ def getApp(app, appId: str): print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId)) return createComposeConfigFromV2(app, nodeRoot) elif 'version' in app and str(app['version']) == "3": - print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.3.0".format(appId)) + print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.3.0".format(appId)) return createComposeConfigFromV3(app, nodeRoot) else: raise Exception("Error: Unsupported version of app.yml") From d6042236d26738cba3a9b0d815243667cd7dd5c2 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 17 Jun 2022 11:29:46 +0000 Subject: [PATCH 08/21] Fix fallback source map entry --- app/lib/manage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/manage.py b/app/lib/manage.py index a4cd44d..55c2c28 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -123,8 +123,8 @@ def getAppYml(name): print("Warning: App {} is not in the source map".format(name), file=sys.stderr) sourceMap = { name: { - "githubRepo": "runcitadel/core", - "branch": "v2" + "githubRepo": "runcitadel/apps", + "branch": "v4-stable" } } url = 'https://raw.githubusercontent.com/{}/{}/apps/{}/app.yml'.format(sourceMap[name]["githubRepo"], sourceMap[name]["branch"], name) From 019fbbc41fe68a396d9445cafa2d908ea3abb0a8 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 17 Jun 2022 11:44:16 +0000 Subject: [PATCH 09/21] Add isQuickUpdate to info.json --- info.json | 1 + 1 file changed, 1 insertion(+) diff --git a/info.json b/info.json index f35a1c9..094bc84 100644 --- a/info.json +++ b/info.json @@ -2,5 +2,6 @@ "version": "0.0.5", "name": "Citadel 0.0.5", "requires": ">=0.0.1", + "isQuickUpdate": false, "notes": "This update fixes a few bugs in the 0.0.4 release that were preventing some apps from working correctly." } From 78cb6d020f517eb587fd11d916a3e7c6bac973f5 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 17 Jun 2022 12:17:38 +0000 Subject: [PATCH 10/21] Fix download URL --- events/triggers/quick-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/events/triggers/quick-update b/events/triggers/quick-update index 999953f..5386e88 100755 --- a/events/triggers/quick-update +++ b/events/triggers/quick-update @@ -12,7 +12,7 @@ cat < "$CITADEL_ROOT"/statuses/update-status.json {"state": "installing", "progress": 30, "description": "Starting update", "updateTo": "$RELEASE"} EOF -curl "https://github.com/runcitadel/core/blob/${RELEASE}/db/dependencies.yml" > "$CITADEL_ROOT"/db/dependencies +curl "https://raw.githubusercontent.com/runcitadel/core/${RELEASE}/db/dependencies.yml" > "$CITADEL_ROOT"/db/dependencies cat < "$CITADEL_ROOT"/statuses/update-status.json {"state": "installing", "progress": 70, "description": "Starting new containers", "updateTo": "$RELEASE"} EOF From faf6d62e42a3bc6fae48d98efef0aa031e9a6662 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sun, 19 Jun 2022 02:05:22 +0200 Subject: [PATCH 11/21] App system cleanups (#51) * Add app cli to docker-compose.yml * Remove app.yml v1 * Add missing import * More cleanups * Another missing import * Add mount for apps * Remove more --- app-standard-v1.json | 230 ------------------ app-standard-v1.json.license | 3 - app/app-standard-v1.yml | 194 --------------- app/lib/composegenerator/shared/env.py | 2 +- app/lib/composegenerator/shared/main.py | 2 +- app/lib/composegenerator/shared/networking.py | 137 +++++++++++ app/lib/composegenerator/v1/generate.py | 30 --- app/lib/composegenerator/v1/networking.py | 226 ----------------- app/lib/composegenerator/v1/types.py | 151 ------------ .../composegenerator/v1/utils/networking.py | 118 --------- app/lib/composegenerator/v2/networking.py | 2 +- app/lib/composegenerator/v3/networking.py | 4 +- app/lib/manage.py | 6 +- app/lib/metadata.py | 2 +- app/lib/validate.py | 12 +- docker-compose.yml | 7 + 16 files changed, 151 insertions(+), 975 deletions(-) delete mode 100644 app-standard-v1.json delete mode 100644 app-standard-v1.json.license delete mode 100644 app/app-standard-v1.yml create mode 100644 app/lib/composegenerator/shared/networking.py delete mode 100644 app/lib/composegenerator/v1/generate.py delete mode 100644 app/lib/composegenerator/v1/networking.py delete mode 100644 app/lib/composegenerator/v1/types.py delete mode 100644 app/lib/composegenerator/v1/utils/networking.py diff --git a/app-standard-v1.json b/app-standard-v1.json deleted file mode 100644 index 0ecb458..0000000 --- a/app-standard-v1.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Citadel app.yml v1", - "description": "The first draft of Citadel's app.yml format", - "type": "object", - "properties": { - "version": { - "type": [ - "string", - "number" - ], - "description": "The version of the app.yml format you're using." - }, - "metadata": { - "type": "object", - "properties": { - "name": { - "description": "Displayed name of the app", - "type": "string" - }, - "version": { - "description": "Displayed version for the app", - "type": "string" - }, - "category": { - "description": "The category you'd put the app in", - "type": "string" - }, - "tagline": { - "description": "A clever tagline", - "type": "string" - }, - "description": { - "description": "A longer description of the app", - "type": "string" - }, - "developer": { - "description": "The awesome people behind the app", - "type": "string" - }, - "website": { - "description": "Displayed version for the app", - "type": "string" - }, - "dependencies": { - "description": "The services the app depends on", - "type": "array", - "items": { - "type": "string" - } - }, - "repo": { - "description": "The development repository for your app", - "type": "string" - }, - "support": { - "description": "A link to the app support wiki/chat/...", - "type": "string" - }, - "gallery": { - "type": "array", - "description": "URLs or paths in the runcitadel/app-images/[app-name] folder with app images", - "items": { - "type": "string" - } - }, - "path": { - "description": "The path of the app's visible site the open button should open", - "type": "string" - }, - "defaultPassword": { - "description": "The app's default password", - "type": "string" - }, - "torOnly": { - "description": "Whether the app is only available over tor", - "type": "boolean" - }, - "mainContainer": { - "type": "string", - "description": "The name of the main container for the app. If set, IP, port, and hidden service will be assigned to it automatically." - }, - "updateContainer": { - "type": "string", - "description": "The container the developer system should automatically update." - } - }, - "required": [ - "name", - "version", - "category", - "tagline", - "description", - "developer", - "website", - "repo", - "support", - "gallery" - ], - "additionalProperties": false - }, - "containers": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "image": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "lnd", - "bitcoind", - "electrum", - "root", - "hw" - ] - } - }, - "ports": { - "type": "array", - "items": { - "type": [ - "string", - "number" - ] - } - }, - "port": { - "type": "number", - "description": "If this is the main container, the port inside the container which will be exposed to the outside as the port specified in metadata." - }, - "environment": { - "type": "object" - }, - "data": { - "type": "array", - "description": "An array of at directories in the container the app stores its data in. Can be empty. Please only list top-level directories.", - "items": { - "type": "string" - } - }, - "user": { - "type": "string", - "description": "The user the container should run as" - }, - "stop_grace_period": { - "type": "string", - "description": "The grace period for stopping the container. Defaults to 1 minute." - }, - "depends_on": { - "type": "array", - "description": "The services the container depends on" - }, - "entrypoint": { - "type": [ - "string", - "array" - ], - "description": "The entrypoint for the container" - }, - "bitcoin_mount_dir": { - "type": "string", - "description": "Where to mount the bitcoin dir" - }, - "command": { - "type": [ - "string", - "array" - ], - "description": "The command for the container" - }, - "init": { - "type": "boolean", - "description": "Whether the container should be run with init" - }, - "stop_signal": { - "type": "string", - "description": "The signal to send to the container when stopping" - }, - "noNetwork": { - "type": "boolean", - "description": "Set this to true if the container shouldn't get an IP & port exposed." - }, - "needsHiddenService": { - "type": "boolean", - "description": "Set this to true if the container should be assigned a hidden service even if it's not the main container." - }, - "hiddenServicePort": { - "type": "number", - "description": "Set this to a port if your container exposes multiple ports, but only one should be a hidden service." - }, - "hiddenServicePorts": { - "type": "object", - "description": "Set this to a map of service names to hidden service ports if your container exposes multiple ports, and all of them should be hidden services.", - "patternProperties": { - "^[a-zA-Z0-9_]+$": { - "type": [ - "number", - "array" - ] - } - } - }, - "restart": { - "type": "string", - "description": "When the container should restart. Can be 'always' or 'on-failure'." - } - }, - "additionalProperties": false, - "required": [ - "name", - "image" - ] - }, - "additionalProperties": false - } - }, - "required": [ - "metadata", - "containers" - ], - "additionalProperties": false -} diff --git a/app-standard-v1.json.license b/app-standard-v1.json.license deleted file mode 100644 index f0626d9..0000000 --- a/app-standard-v1.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2021 Citadel and contributors - -SPDX-License-Identifier: GPL-3.0-or-later \ No newline at end of file diff --git a/app/app-standard-v1.yml b/app/app-standard-v1.yml deleted file mode 100644 index 3000a25..0000000 --- a/app/app-standard-v1.yml +++ /dev/null @@ -1,194 +0,0 @@ -# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema -$schema: https://json-schema.org/draft/2020-12/schema - - -title: Citadel app.yml v1 -description: The first draft of Citadel's app.yml format -type: object - -properties: - version: - type: - - string - - number - description: The version of the app.yml format you're using. - - metadata: - type: object - properties: - name: - description: Displayed name of the app - type: string - version: - description: Displayed version for the app - type: string - category: - description: The category you'd put the app in - type: string - tagline: - description: A clever tagline - type: string - description: - description: A longer description of the app - type: string - developer: - description: The awesome people behind the app - type: string - website: - description: Displayed version for the app - type: string - dependencies: - description: The services the app depends on - type: array - items: - type: string - repo: - description: The development repository for your app - type: string - support: - description: A link to the app support wiki/chat/... - type: string - gallery: - type: array - description: >- - URLs or paths in the runcitadel/app-images/[app-name] folder with app - images - items: - type: string - path: - description: The path of the app's visible site the open button should open - type: string - defaultPassword: - description: The app's default password - type: string - torOnly: - description: Whether the app is only available over tor - type: boolean - mainContainer: - type: string - description: >- - The name of the main container for the app. If set, IP, port, and - hidden service will be assigned to it automatically. - updateContainer: - type: string - description: The container the developer system should automatically update. - required: - - name - - version - - category - - tagline - - description - - developer - - website - - repo - - support - - gallery - additionalProperties: false - - containers: - type: array - items: - type: object - properties: - name: - type: string - image: - type: string - permissions: - type: array - items: - type: string - enum: - - lnd - - bitcoind - - electrum - - root - - hw - ports: - type: array - items: - type: - - string - - number - port: - type: number - description: >- - If this is the main container, the port inside the container which - will be exposed to the outside as the port specified in metadata. - environment: - type: object - data: - type: array - description: >- - An array of at directories in the container the app stores its data - in. Can be empty. Please only list top-level directories. - items: - type: string - user: - type: string - description: The user the container should run as - stop_grace_period: - type: string - description: The grace period for stopping the container. Defaults to 1 minute. - depends_on: - type: array - description: The services the container depends on - entrypoint: - type: - - string - - array - description: The entrypoint for the container - bitcoin_mount_dir: - type: string - description: Where to mount the bitcoin dir - command: - type: - - string - - array - description: The command for the container - init: - type: boolean - description: Whether the container should be run with init - stop_signal: - type: string - description: The signal to send to the container when stopping - noNetwork: - type: boolean - description: >- - Set this to true if the container shouldn't get an IP & port - exposed. - needsHiddenService: - type: boolean - description: >- - Set this to true if the container should be assigned a hidden - service even if it's not the main container. - hiddenServicePort: - type: number - description: >- - Set this to a port if your container exposes multiple ports, but - only one should be a hidden service. - hiddenServicePorts: - type: object - description: >- - Set this to a map of service names to hidden service ports if your - container exposes multiple ports, and all of them should be hidden - services. - patternProperties: - ^[a-zA-Z0-9_]+$: - type: - - number - - array - restart: - type: string - description: When the container should restart. Can be 'always' or 'on-failure'. - additionalProperties: false - required: - - name - - image - additionalProperties: false - -required: - - metadata - - containers - -additionalProperties: false diff --git a/app/lib/composegenerator/shared/env.py b/app/lib/composegenerator/shared/env.py index 69b4318..fb12f1a 100644 --- a/app/lib/composegenerator/shared/env.py +++ b/app/lib/composegenerator/shared/env.py @@ -4,7 +4,7 @@ import re from typing import Union -from lib.composegenerator.v1.types import App +from lib.composegenerator.v2.types import App from lib.composegenerator.shared.const import always_allowed_env from lib.citadelutils import checkArrayContainsAllElements, getEnvVars diff --git a/app/lib/composegenerator/shared/main.py b/app/lib/composegenerator/shared/main.py index 4cc14df..da3c553 100644 --- a/app/lib/composegenerator/shared/main.py +++ b/app/lib/composegenerator/shared/main.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # Main functions -from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container +from lib.composegenerator.v2.types import App, AppStage3, AppStage2, Container from lib.composegenerator.shared.const import permissions diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py new file mode 100644 index 0000000..0b4e47e --- /dev/null +++ b/app/lib/composegenerator/shared/networking.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +import json +from os import path +import random +from lib.composegenerator.v2.types import ContainerStage2, NetworkConfig +from lib.citadelutils import parse_dotenv +from dacite import from_dict + +def getFreePort(networkingFile: str, appId: str): + # Ports used currently in Citadel + usedPorts = [ + # Dashboard + 80, + # Sometimes used by nginx with some setups + 433, + # Dashboard SSL + 443, + # Bitcoin Core P2P + 8333, + # LND gRPC + 10009, + # LND REST + 8080, + # Electrum Server + 50001, + # Tor Proxy + 9050, + ] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + if 'ports' in networkingData: + usedPorts += list(networkingData['ports'].values()) + else: + networkingData['ports'] = {} + + if appId in networkingData['ports']: + return networkingData['ports'][appId] + + while True: + port = str(random.randint(1024, 49151)) + if port not in usedPorts: + # Check if anyhing is listening on the specific port + if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: + networkingData['ports'][appId] = port + break + + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + + return port + +def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1: + raise Exception("Newline in container name") + env_var = "APP_{}_{}_IP".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP + # can be used + usedIps = [] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + + if 'ip_addresses' in networkingData: + usedIps = list(networkingData['ip_addresses'].values()) + else: + networkingData['ip_addresses'] = {} + # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container + # If the IP is already in use, it will be tried again until it's not in use + # If it's not in use, it will be added to the usedIps list and written to the usedIpFile + # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), + # Throw an error, because no more IPs can be used + if len(usedIps) == 235: + raise Exception("No more IPs can be used") + + if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: + ip = networkingData['ip_addresses']["{}-{}".format( + appId, container.name)] + else: + while True: + ip = "10.21.21." + str(random.randint(20, 255)) + if ip not in usedIps: + networkingData['ip_addresses']["{}-{}".format( + appId, container.name)] = ip + break + container.networks = from_dict(data_class=NetworkConfig, data={'default': { + 'ipv4_address': "$" + env_var}}) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): + return container + + # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, ip)) + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + return container + + +def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1 or container.name.find(" ") != -1: + raise Exception("Newline or space in container name") + + env_var = "APP_{}_{}_PORT".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + + port = getFreePort(networkingFile, appId) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(port): + return {"port": port, "env_var": "${{{}}}".format(env_var)} + + # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, port)) + + # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} + # where the outer {{ }} will be replaced by {} in the returned string + return {"port": port, "env_var": "${{{}}}".format(env_var)} + diff --git a/app/lib/composegenerator/v1/generate.py b/app/lib/composegenerator/v1/generate.py deleted file mode 100644 index fa01fd4..0000000 --- a/app/lib/composegenerator/v1/generate.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from lib.composegenerator.v1.types import App, AppStage4, generateApp -from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices -from lib.composegenerator.shared.env import validateEnv -from lib.citadelutils import classToDict -import os - -def createComposeConfigFromV1(app: dict, nodeRoot: str): - envFile = os.path.join(nodeRoot, ".env") - networkingFile = os.path.join(nodeRoot, "apps", "networking.json") - - newApp: App = generateApp(app) - newApp = convertContainerPermissions(newApp) - validateEnv(newApp) - newApp = convertDataDirToVolume(newApp) - newApp = configureIps(newApp, networkingFile, envFile) - newApp = configureMainPort(newApp, nodeRoot) - configureHiddenServices(newApp, nodeRoot) - finalConfig: AppStage4 = convertContainersToServices(newApp) - newApp = classToDict(finalConfig) - del newApp['metadata'] - if "version" in newApp: - del newApp["version"] - # Set version to 3.8 (current compose file version) - newApp = {'version': '3.8', **newApp} - return newApp diff --git a/app/lib/composegenerator/v1/networking.py b/app/lib/composegenerator/v1/networking.py deleted file mode 100644 index 68d1dc1..0000000 --- a/app/lib/composegenerator/v1/networking.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from dacite import from_dict -from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig -from lib.citadelutils import parse_dotenv -import json -from os import path -import random -from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService - - -def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1: - raise Exception("Newline in container name") - env_var = "APP_{}_{}_IP".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP - # can be used - usedIps = [] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - - if 'ip_addresses' in networkingData: - usedIps = list(networkingData['ip_addresses'].values()) - else: - networkingData['ip_addresses'] = {} - # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container - # If the IP is already in use, it will be tried again until it's not in use - # If it's not in use, it will be added to the usedIps list and written to the usedIpFile - # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), - # Throw an error, because no more IPs can be used - if len(usedIps) == 235: - raise Exception("No more IPs can be used") - - if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: - ip = networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] - else: - while True: - ip = "10.21.21." + str(random.randint(20, 255)) - if ip not in usedIps: - networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] = ip - break - container.networks = from_dict(data_class=NetworkConfig, data={'default': { - 'ipv4_address': "$" + env_var}}) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): - return container - - # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, ip)) - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - return container - - -def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1 or container.name.find(" ") != -1: - raise Exception("Newline or space in container name") - - env_var = "APP_{}_{}_PORT".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - - port = getFreePort(networkingFile, appId) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(port): - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, port)) - - # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} - # where the outer {{ }} will be replaced by {} in the returned string - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - -def getMainContainer(app: dict): - if len(app.containers) == 1: - return app.containers[0] - else: - if not app.metadata.mainContainer: - app.metadata.mainContainer = 'main' - for container in app.containers: - if container.name == app.metadata.mainContainer: - return container - raise Exception( - "No main container found for app {}".format(app.metadata.name)) - - -def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: - registryFile = path.join(nodeRoot, "apps", "registry.json") - registry: list = [] - if path.isfile(registryFile): - with open(registryFile, 'r') as f: - registry = json.load(f) - else: - raise Exception("Registry file not found") - - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - - mainContainer = getMainContainer(app) - - portDetails = assignPort(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - containerPort = portDetails['port'] - portAsEnvVar = portDetails['env_var'] - portToAppend = portAsEnvVar - - mainPort = False - - if mainContainer.port: - portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port) - mainPort = mainContainer.port - del mainContainer.port - else: - portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar) - - if mainContainer.ports: - mainContainer.ports.append(portToAppend) - # Set the main port to the first port in the list, if it contains a :, it's the port after the : - # If it doesn't contain a :, it's the port itself - if mainPort == False: - mainPort = mainContainer.ports[0] - if mainPort.find(":") != -1: - mainPort = mainPort.split(":")[1] - else: - mainContainer.ports = [portToAppend] - if mainPort == False: - mainPort = portDetails['port'] - - mainContainer = assignIp(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - - # If the IP wasn't in dotenv before, now it should be - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - - containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace( - "-", "_"), mainContainer.name.upper().replace("-", "_"))] - - hiddenservice = getHiddenService( - app.metadata.name, app.metadata.id, containerIP, mainPort) - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenservice) - - # Also set the port in metadata - app.metadata.port = int(containerPort) - - for registryApp in registry: - if registryApp['id'] == app.metadata.id: - registry[registry.index(registryApp)]['port'] = int(containerPort) - break - - with open(registryFile, 'w') as f: - json.dump(registry, f, indent=4, sort_keys=True) - - return app - - -def configureIps(app: AppStage2, networkingFile: str, envFile: str): - for container in app.containers: - if container.noNetwork: - # Check if port is defined for the container - if container.port: - raise Exception("Port defined for container without network") - if app.metadata.mainContainer == container.name: - raise Exception("Main container without network") - # Skip this iteration of the loop - continue - - container = assignIp(container, app.metadata.id, - networkingFile, envFile) - - return app - - -def configureHiddenServices(app: dict, nodeRoot: str) -> None: - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - hiddenServices = "" - - if len(app.containers) == 1: - mainContainer = app.containers[0] - else: - mainContainer = None - if app.metadata.mainContainer == None: - app.metadata.mainContainer = 'main' - for container in app.containers: - if container.name == app.metadata.mainContainer: - mainContainer = container - break - if mainContainer is None: - raise Exception("No main container found") - - for container in app.containers: - env_var = "APP_{}_{}_IP".format( - app.metadata.id.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - hiddenServices += getContainerHiddenService( - app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name) - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenServices) diff --git a/app/lib/composegenerator/v1/types.py b/app/lib/composegenerator/v1/types.py deleted file mode 100644 index c9556be..0000000 --- a/app/lib/composegenerator/v1/types.py +++ /dev/null @@ -1,151 +0,0 @@ -from typing import Union, List -from dataclasses import dataclass, field -from dacite import from_dict - -@dataclass -class Metadata: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - repo: str - support: str - gallery: List[str] = field(default_factory=list) - dependencies: List[str] = field(default_factory=list) - mainContainer: Union[str, None] = None - updateContainer: Union[str, None] = None - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - -@dataclass -class Container: - name: str - image: str - permissions: list = field(default_factory=list) - ports: list = field(default_factory=list) - port: Union[int, None] = None - environment: Union[dict, None] = None - data: list = field(default_factory=list) - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: list = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - bitcoin_mount_dir: Union[str, None] = None - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - environment_allow: list = field(default_factory=list) - # Only added later - volumes: list = field(default_factory=list) - restart: Union[str, None] = None - -@dataclass -class App: - version: Union[str, int] - metadata: Metadata - containers: List[Container] - -# Generate an app instance from an app dict -def generateApp(appDict): - return from_dict(data_class=App, data=appDict) - -@dataclass -class Network: - ipv4_address: Union[str, None] = None - -@dataclass -class NetworkConfig: - default: Network - -# After converting data dir and defining volumes, stage 2 -@dataclass -class ContainerStage2: - id: str - name: str - image: str - permissions: List[str] = field(default_factory=list) - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - -@dataclass -class AppStage2: - version: Union[str, int] - metadata: Metadata - containers: List[ContainerStage2] - -@dataclass -class MetadataStage3: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - dependencies: List[str] - repo: str - support: str - gallery: List[str] - mainContainer: Union[str, None] = None - updateContainer: Union[str, None] = None - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - -@dataclass -class AppStage3: - version: Union[str, int] - metadata: MetadataStage3 - containers: List[ContainerStage2] - -@dataclass -class ContainerStage4: - id: str - name: str - image: str - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - -@dataclass -class AppStage4: - version: Union[str, int] - metadata: MetadataStage3 - services: List[ContainerStage4] \ No newline at end of file diff --git a/app/lib/composegenerator/v1/utils/networking.py b/app/lib/composegenerator/v1/utils/networking.py deleted file mode 100644 index 2668aa7..0000000 --- a/app/lib/composegenerator/v1/utils/networking.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import json -import os -import random - -from lib.composegenerator.v1.types import Container - -def getFreePort(networkingFile: str, appId: str): - # Ports used currently in Citadel - usedPorts = [ - # Dashboard - 80, - # Sometimes used by nginx with some setups - 433, - # Dashboard SSL - 443, - # Bitcoin Core P2P - 8333, - # LND gRPC - 10009, - # LND REST - 8080, - # Electrum Server - 50001, - # Tor Proxy - 9050, - ] - networkingData = {} - if os.path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - if 'ports' in networkingData: - usedPorts += list(networkingData['ports'].values()) - else: - networkingData['ports'] = {} - - if appId in networkingData['ports']: - return networkingData['ports'][appId] - - while True: - port = str(random.randint(1024, 49151)) - if port not in usedPorts: - # Check if anyhing is listening on the specific port - if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: - networkingData['ports'][appId] = port - break - - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - - return port - - -def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str: - hiddenServices = ''' -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -'''.format(name, id) - for port in ports: - hiddenServices += 'HiddenServicePort {} {}:{}'.format( - port, internalIp, port) - hiddenServices += "\n" - return hiddenServices - - -def getHiddenServiceString(name: str, id: str, internalPort, internalIp: str, publicPort) -> str: - return ''' -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -HiddenServicePort {} {}:{} - -'''.format(name, id, publicPort, internalIp, internalPort) - - -def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str: - return getHiddenServiceString(appName, appId, appPort, appIp, "80") - - -def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str: - if not container.needsHiddenService and not isMainContainer: - return "" - if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer: - print("Container {} for app {} isn't compatible with hidden service assignment".format( - container.name, appName)) - return "" - - if isMainContainer: - if not container.hiddenServicePorts: - return "" - # hiddenServicePorts is a map of hidden service name to port - # We need to generate a hidden service for each one - hiddenServices = "" - for name, port in container.hiddenServicePorts.items(): - if ".." in name: - print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName)) - - # If port is a list, use getHiddenServiceMultiPort - if isinstance(port, list): - hiddenServices += getHiddenServiceMultiPort("{} {}".format(appName, name), "{}-{}".format( - appId, name), containerIp, port) - else: - hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format( - appId, name), port, containerIp, port) - del container.hiddenServicePorts - return hiddenServices - - del container.needsHiddenService - if not container.port: - data = getHiddenServiceString(appName + container.name, "{}-{}".format( - appId, container.name), container.hiddenServicePort, containerIp, "80") - del container.hiddenServicePort - return data - else: - return getHiddenServiceString(appName + container.name, "{}-{}".format( - appId, container.name), container.port, containerIp, container.port) diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py index ec5d4ad..a69540d 100644 --- a/app/lib/composegenerator/v2/networking.py +++ b/app/lib/composegenerator/v2/networking.py @@ -8,7 +8,7 @@ import json from os import path import random from lib.composegenerator.v2.utils.networking import getContainerHiddenService -from lib.composegenerator.v1.networking import assignIp, assignPort +from lib.composegenerator.shared.networking import assignIp, assignPort def getMainContainer(app: App) -> Container: diff --git a/app/lib/composegenerator/v3/networking.py b/app/lib/composegenerator/v3/networking.py index 66d5a96..2e25f55 100644 --- a/app/lib/composegenerator/v3/networking.py +++ b/app/lib/composegenerator/v3/networking.py @@ -7,8 +7,7 @@ from lib.citadelutils import parse_dotenv import json from os import path import random -from lib.composegenerator.v1.networking import assignIp, assignPort - +from lib.composegenerator.shared.networking import assignIp, assignPort def getMainContainerIndex(app: App): if len(app.containers) == 1: @@ -105,4 +104,3 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: with open(envFile, 'a') as f: f.write("{}={}\n".format(portAsEnvVar, app.metadata.port)) return app - diff --git a/app/lib/manage.py b/app/lib/manage.py index 3aabd5e..3033bec 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -25,7 +25,6 @@ except Exception: print("Continuing anyway, but some features won't be available,") print("for example checking for app updates") -from lib.composegenerator.v1.generate import createComposeConfigFromV1 from lib.composegenerator.v2.generate import createComposeConfigFromV2 from lib.composegenerator.v3.generate import createComposeConfigFromV3 from lib.validate import findAndValidateApps @@ -188,10 +187,7 @@ def getApp(appFile: str, appId: str): raise Exception("Error: Could not find metadata in " + appFile) app["metadata"]["id"] = appId - if 'version' in app and str(app['version']) == "1": - print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId)) - return createComposeConfigFromV1(app, nodeRoot) - elif 'version' in app and str(app['version']) == "2": + if 'version' in app and str(app['version']) == "2": print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId)) return createComposeConfigFromV2(app, nodeRoot) elif 'version' in app and str(app['version']) == "3": diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 6db5672..4ab7ca1 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -7,7 +7,7 @@ import yaml from lib.composegenerator.next.stage1 import createCleanConfigFromV3 from lib.composegenerator.v2.networking import getMainContainer -from lib.composegenerator.v1.networking import getFreePort +from lib.composegenerator.shared.networking import getFreePort from lib.entropy import deriveEntropy from typing import List import json diff --git a/app/lib/validate.py b/app/lib/validate.py index 9dfe236..16009d6 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -9,8 +9,6 @@ import yaml scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") -with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f: - schemaVersion1 = yaml.safe_load(f) with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: schemaVersion2 = yaml.safe_load(f) with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: @@ -19,15 +17,7 @@ with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: # Validates app data # Returns true if valid, false otherwise def validateApp(app: dict): - if 'version' in app and str(app['version']) == "1": - try: - validate(app, schemaVersion1) - return True - # Catch and log any errors, and return false - except Exception as e: - print(e) - return False - elif 'version' in app and str(app['version']) == "2": + if 'version' in app and str(app['version']) == "2": try: validate(app, schemaVersion2) return True diff --git a/docker-compose.yml b/docker-compose.yml index 60eb321..a83901b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -234,6 +234,13 @@ services: networks: default: ipv4_address: $REDIS_IP + + app-cli: + container_name: app-cli + image: ghcr.io/runcitadel/app-cli:main@sha256:694e52fa9da1ac976165f269c17e27803032a05a76293dfe3589a50813306ded + volumes: + - ${PWD}/apps:/apps + networks: default: name: citadel_main_network From b9f85c3ca328b07f06e3ad252c053d2176aa06b3 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Sun, 19 Jun 2022 14:19:12 +0000 Subject: [PATCH 12/21] More logging --- app/lib/validate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/lib/validate.py b/app/lib/validate.py index b2c8b9b..97a9340 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -72,6 +72,8 @@ def findAndValidateApps(dir: str): # Read the app.yml and append it to app_data with open(os.path.join(app_dir, "app.yml"), 'r') as f: app_data[name] = yaml.safe_load(f) + else: + print("App {} has no app.yml".format(name)) # Now validate all the apps using the validateAppFile function by passing the app.yml as an argument to it, if an app is invalid, remove it from the list for app in apps: appyml = app_data[app] From 21b5c529c6de7176a1947a7668afc8f6cdc755c5 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Sun, 19 Jun 2022 14:26:15 +0000 Subject: [PATCH 13/21] Simplify listing apps --- app/lib/validate.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/app/lib/validate.py b/app/lib/validate.py index 97a9340..66e6a33 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -64,16 +64,18 @@ def findApps(dir: str): def findAndValidateApps(dir: str): apps = [] app_data = {} - for root, dirs, files in os.walk(dir, topdown=False): - for name in dirs: - app_dir = os.path.join(root, name) - if os.path.isfile(os.path.join(app_dir, "app.yml")): - apps.append(name) - # Read the app.yml and append it to app_data - with open(os.path.join(app_dir, "app.yml"), 'r') as f: - app_data[name] = yaml.safe_load(f) - else: - print("App {} has no app.yml".format(name)) + for subdir in os.scandir(dir): + if not subdir.is_dir(): + continue + + app_dir = subdir.path + if os.path.isfile(os.path.join(app_dir, "app.yml")): + apps.append(name) + # Read the app.yml and append it to app_data + with open(os.path.join(app_dir, "app.yml"), 'r') as f: + app_data[subdir.name] = yaml.safe_load(f) + else: + print("App {} has no app.yml".format(subdir.name)) # Now validate all the apps using the validateAppFile function by passing the app.yml as an argument to it, if an app is invalid, remove it from the list for app in apps: appyml = app_data[app] From 58700fc5817d71fe312db54f7153b9407c007fb8 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Sun, 19 Jun 2022 14:28:29 +0000 Subject: [PATCH 14/21] Minor fix --- app/lib/validate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/lib/validate.py b/app/lib/validate.py index 66e6a33..13f77f8 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -67,10 +67,9 @@ def findAndValidateApps(dir: str): for subdir in os.scandir(dir): if not subdir.is_dir(): continue - app_dir = subdir.path if os.path.isfile(os.path.join(app_dir, "app.yml")): - apps.append(name) + apps.append(subdir.name) # Read the app.yml and append it to app_data with open(os.path.join(app_dir, "app.yml"), 'r') as f: app_data[subdir.name] = yaml.safe_load(f) From c4de7d10aaa2e06adeb38f8aeff1320c04b6d71b Mon Sep 17 00:00:00 2001 From: nolim1t - f6287b82CC84bcbd Date: Mon, 20 Jun 2022 20:55:27 +0700 Subject: [PATCH 15/21] Update Tor to 0.4.7.8 (#58) --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a83901b..a6d7883 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: tor: container_name: tor - image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227 + image: lncm/tor:0.4.7.8@sha256:aab30ebb496aa25934d6096951d8b200347c3c3ce5db3493695229efa2601f7b user: toruser restart: on-failure volumes: @@ -15,7 +15,7 @@ services: ipv4_address: $TOR_PROXY_IP app-tor: container_name: app-tor - image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227 + image: lncm/tor:0.4.7.8@sha256:aab30ebb496aa25934d6096951d8b200347c3c3ce5db3493695229efa2601f7b user: toruser restart: on-failure volumes: @@ -26,7 +26,7 @@ services: ipv4_address: $APPS_TOR_IP app-2-tor: container_name: app-2-tor - image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227 + image: lncm/tor:0.4.7.8@sha256:aab30ebb496aa25934d6096951d8b200347c3c3ce5db3493695229efa2601f7b user: toruser restart: on-failure volumes: @@ -37,7 +37,7 @@ services: ipv4_address: $APPS_2_TOR_IP app-3-tor: container_name: app-3-tor - image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227 + image: lncm/tor:0.4.7.8@sha256:aab30ebb496aa25934d6096951d8b200347c3c3ce5db3493695229efa2601f7b user: toruser restart: on-failure volumes: From 35d92828c4a9ea14d691e1fd76f46058887cd7de Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 21 Jun 2022 09:14:22 +0000 Subject: [PATCH 16/21] Fix redis server permissions --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5dead34..aa8b71c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -223,6 +223,7 @@ services: ipv4_address: $ELECTRUM_IP redis: container_name: redis + user: 1000:1000 image: redis:7.0.0-bullseye@sha256:ad0705f2e2344c4b642449e658ef4669753d6eb70228d46267685045bf932303 working_dir: /data volumes: From ddf7861706ce189307152f890b38a9cba81846de Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Tue, 21 Jun 2022 09:15:10 +0000 Subject: [PATCH 17/21] Fix set-update-channel help text --- scripts/set-update-channel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/set-update-channel b/scripts/set-update-channel index a917095..1495702 100755 --- a/scripts/set-update-channel +++ b/scripts/set-update-channel @@ -10,7 +10,7 @@ NODE_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)" # If $1 is not given, fail if [ -z "$1" ]; then echo "Usage: $0 " - echo "Channel can currently either be 'stable' or 'beta'" + echo "Channel can currently either be 'stable', 'beta' or 'c-lightning'" exit 1 fi sed -i "s/UPDATE_CHANNEL=.*/UPDATE_CHANNEL=${1}/" "${NODE_ROOT}/.env" From f75ba590bceb4e5d57fd3423f9a7d92de488a82c Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 24 Jun 2022 08:17:56 +0000 Subject: [PATCH 18/21] Update containers --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index aa8b71c..c18b677 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,7 @@ services: ipv4_address: $LND_IP dashboard: container_name: dashboard - image: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5 + image: ghcr.io/runcitadel/dashboard:fix-small-issues-and-translations@sha256:82059a11e50d77eec8edb20a8c0e1f821988ea176b928944738df1a93304c26e restart: on-failure stop_grace_period: 1m30s networks: @@ -108,7 +108,7 @@ services: ipv4_address: $DASHBOARD_IP manager: container_name: manager - image: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce + image: ghcr.io/runcitadel/manager:v0.0.16@sha256:9de3acad76e957811ff7f81cdba8b9a280a79762d24e1bcd16a4517e5643dfca depends_on: - tor - redis From 0db1b1ddb1f9a16564e6a82bd995edb961c07b09 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Fri, 24 Jun 2022 08:21:39 +0000 Subject: [PATCH 19/21] Update containers in dependencies.yml --- db/dependencies.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/dependencies.yml b/db/dependencies.yml index dbdd96a..df11e91 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -1,5 +1,5 @@ compose: v2.6.0 -dashboard: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5 -manager: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce +dashboard: ghcr.io/runcitadel/dashboard:fix-small-issues-and-translations@sha256:82059a11e50d77eec8edb20a8c0e1f821988ea176b928944738df1a93304c26e +manager: ghcr.io/runcitadel/manager:v0.0.16@sha256:9de3acad76e957811ff7f81cdba8b9a280a79762d24e1bcd16a4517e5643dfca middleware: ghcr.io/runcitadel/middleware:v0.0.11@sha256:e472da8cbfa67d9a9dbf321334fe65cdf20a0f9b6d6bab33fdf07210f54e7002 app-cli: ghcr.io/runcitadel/app-cli:main@sha256:f532923eac28cfac03579cbb440397bcf16c8730f291b39eeada8278331f7054 From fb7a25b9dd1aa787ed000033673146e485356ab7 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Sat, 25 Jun 2022 09:33:26 +0000 Subject: [PATCH 20/21] Fix sources.list --- app-system/sources.list | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app-system/sources.list b/app-system/sources.list index 2466e50..b8c131a 100644 --- a/app-system/sources.list +++ b/app-system/sources.list @@ -3,9 +3,9 @@ # SPDX-License-Identifier: GPL-3.0-or-later # A collection of fully FLOSS app definitions and FLOSS apps for Citadel. -https://github.com/runcitadel/apps v3-stable +https://github.com/runcitadel/apps v4-dev # Some apps modified version of Umbrel apps, and their app definitions aren't FLOSS yet. # Include them anyway, but as a separate repo. # Add a # to the line below to disable the repo and only use FLOSS apps. -https://github.com/runcitadel/apps-nonfree v3-stable +https://github.com/runcitadel/apps-nonfree v3-beta From c896545790adebc4b3e2a7891d3563155f58bc60 Mon Sep 17 00:00:00 2001 From: AaronDewes Date: Mon, 11 Jul 2022 09:13:21 +0000 Subject: [PATCH 21/21] Pull container early --- app/lib/manage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/lib/manage.py b/app/lib/manage.py index 55c2c28..d6235ec 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -144,6 +144,7 @@ def update(verbose: bool = False): json.dump(registry["ports"], f, sort_keys=True) print("Wrote registry to registry.json") + os.system("docker pull {}".format(dependencies['app-cli'])) threads = list() # Loop through the apps and generate valid compose files from them, then put these into the app dir for app in apps: