From faf6d62e42a3bc6fae48d98efef0aa031e9a6662 Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Sun, 19 Jun 2022 02:05:22 +0200 Subject: [PATCH] App system cleanups (#51) * Add app cli to docker-compose.yml * Remove app.yml v1 * Add missing import * More cleanups * Another missing import * Add mount for apps * Remove more --- app-standard-v1.json | 230 ------------------ app-standard-v1.json.license | 3 - app/app-standard-v1.yml | 194 --------------- app/lib/composegenerator/shared/env.py | 2 +- app/lib/composegenerator/shared/main.py | 2 +- app/lib/composegenerator/shared/networking.py | 137 +++++++++++ app/lib/composegenerator/v1/generate.py | 30 --- app/lib/composegenerator/v1/networking.py | 226 ----------------- app/lib/composegenerator/v1/types.py | 151 ------------ .../composegenerator/v1/utils/networking.py | 118 --------- app/lib/composegenerator/v2/networking.py | 2 +- app/lib/composegenerator/v3/networking.py | 4 +- app/lib/manage.py | 6 +- app/lib/metadata.py | 2 +- app/lib/validate.py | 12 +- docker-compose.yml | 7 + 16 files changed, 151 insertions(+), 975 deletions(-) delete mode 100644 app-standard-v1.json delete mode 100644 app-standard-v1.json.license delete mode 100644 app/app-standard-v1.yml create mode 100644 app/lib/composegenerator/shared/networking.py delete mode 100644 app/lib/composegenerator/v1/generate.py delete mode 100644 app/lib/composegenerator/v1/networking.py delete mode 100644 app/lib/composegenerator/v1/types.py delete mode 100644 app/lib/composegenerator/v1/utils/networking.py diff --git a/app-standard-v1.json b/app-standard-v1.json deleted file mode 100644 index 0ecb458..0000000 --- a/app-standard-v1.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Citadel app.yml v1", - "description": "The first draft of Citadel's app.yml format", - "type": "object", - "properties": { - "version": { - "type": [ - "string", - "number" - ], - "description": "The version of the app.yml format you're using." - }, - "metadata": { - "type": "object", - "properties": { - "name": { - "description": "Displayed name of the app", - "type": "string" - }, - "version": { - "description": "Displayed version for the app", - "type": "string" - }, - "category": { - "description": "The category you'd put the app in", - "type": "string" - }, - "tagline": { - "description": "A clever tagline", - "type": "string" - }, - "description": { - "description": "A longer description of the app", - "type": "string" - }, - "developer": { - "description": "The awesome people behind the app", - "type": "string" - }, - "website": { - "description": "Displayed version for the app", - "type": "string" - }, - "dependencies": { - "description": "The services the app depends on", - "type": "array", - "items": { - "type": "string" - } - }, - "repo": { - "description": "The development repository for your app", - "type": "string" - }, - "support": { - "description": "A link to the app support wiki/chat/...", - "type": "string" - }, - "gallery": { - "type": "array", - "description": "URLs or paths in the runcitadel/app-images/[app-name] folder with app images", - "items": { - "type": "string" - } - }, - "path": { - "description": "The path of the app's visible site the open button should open", - "type": "string" - }, - "defaultPassword": { - "description": "The app's default password", - "type": "string" - }, - "torOnly": { - "description": "Whether the app is only available over tor", - "type": "boolean" - }, - "mainContainer": { - "type": "string", - "description": "The name of the main container for the app. If set, IP, port, and hidden service will be assigned to it automatically." - }, - "updateContainer": { - "type": "string", - "description": "The container the developer system should automatically update." - } - }, - "required": [ - "name", - "version", - "category", - "tagline", - "description", - "developer", - "website", - "repo", - "support", - "gallery" - ], - "additionalProperties": false - }, - "containers": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "image": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "lnd", - "bitcoind", - "electrum", - "root", - "hw" - ] - } - }, - "ports": { - "type": "array", - "items": { - "type": [ - "string", - "number" - ] - } - }, - "port": { - "type": "number", - "description": "If this is the main container, the port inside the container which will be exposed to the outside as the port specified in metadata." - }, - "environment": { - "type": "object" - }, - "data": { - "type": "array", - "description": "An array of at directories in the container the app stores its data in. Can be empty. Please only list top-level directories.", - "items": { - "type": "string" - } - }, - "user": { - "type": "string", - "description": "The user the container should run as" - }, - "stop_grace_period": { - "type": "string", - "description": "The grace period for stopping the container. Defaults to 1 minute." - }, - "depends_on": { - "type": "array", - "description": "The services the container depends on" - }, - "entrypoint": { - "type": [ - "string", - "array" - ], - "description": "The entrypoint for the container" - }, - "bitcoin_mount_dir": { - "type": "string", - "description": "Where to mount the bitcoin dir" - }, - "command": { - "type": [ - "string", - "array" - ], - "description": "The command for the container" - }, - "init": { - "type": "boolean", - "description": "Whether the container should be run with init" - }, - "stop_signal": { - "type": "string", - "description": "The signal to send to the container when stopping" - }, - "noNetwork": { - "type": "boolean", - "description": "Set this to true if the container shouldn't get an IP & port exposed." - }, - "needsHiddenService": { - "type": "boolean", - "description": "Set this to true if the container should be assigned a hidden service even if it's not the main container." - }, - "hiddenServicePort": { - "type": "number", - "description": "Set this to a port if your container exposes multiple ports, but only one should be a hidden service." - }, - "hiddenServicePorts": { - "type": "object", - "description": "Set this to a map of service names to hidden service ports if your container exposes multiple ports, and all of them should be hidden services.", - "patternProperties": { - "^[a-zA-Z0-9_]+$": { - "type": [ - "number", - "array" - ] - } - } - }, - "restart": { - "type": "string", - "description": "When the container should restart. Can be 'always' or 'on-failure'." - } - }, - "additionalProperties": false, - "required": [ - "name", - "image" - ] - }, - "additionalProperties": false - } - }, - "required": [ - "metadata", - "containers" - ], - "additionalProperties": false -} diff --git a/app-standard-v1.json.license b/app-standard-v1.json.license deleted file mode 100644 index f0626d9..0000000 --- a/app-standard-v1.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2021 Citadel and contributors - -SPDX-License-Identifier: GPL-3.0-or-later \ No newline at end of file diff --git a/app/app-standard-v1.yml b/app/app-standard-v1.yml deleted file mode 100644 index 3000a25..0000000 --- a/app/app-standard-v1.yml +++ /dev/null @@ -1,194 +0,0 @@ -# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema -$schema: https://json-schema.org/draft/2020-12/schema - - -title: Citadel app.yml v1 -description: The first draft of Citadel's app.yml format -type: object - -properties: - version: - type: - - string - - number - description: The version of the app.yml format you're using. - - metadata: - type: object - properties: - name: - description: Displayed name of the app - type: string - version: - description: Displayed version for the app - type: string - category: - description: The category you'd put the app in - type: string - tagline: - description: A clever tagline - type: string - description: - description: A longer description of the app - type: string - developer: - description: The awesome people behind the app - type: string - website: - description: Displayed version for the app - type: string - dependencies: - description: The services the app depends on - type: array - items: - type: string - repo: - description: The development repository for your app - type: string - support: - description: A link to the app support wiki/chat/... - type: string - gallery: - type: array - description: >- - URLs or paths in the runcitadel/app-images/[app-name] folder with app - images - items: - type: string - path: - description: The path of the app's visible site the open button should open - type: string - defaultPassword: - description: The app's default password - type: string - torOnly: - description: Whether the app is only available over tor - type: boolean - mainContainer: - type: string - description: >- - The name of the main container for the app. If set, IP, port, and - hidden service will be assigned to it automatically. - updateContainer: - type: string - description: The container the developer system should automatically update. - required: - - name - - version - - category - - tagline - - description - - developer - - website - - repo - - support - - gallery - additionalProperties: false - - containers: - type: array - items: - type: object - properties: - name: - type: string - image: - type: string - permissions: - type: array - items: - type: string - enum: - - lnd - - bitcoind - - electrum - - root - - hw - ports: - type: array - items: - type: - - string - - number - port: - type: number - description: >- - If this is the main container, the port inside the container which - will be exposed to the outside as the port specified in metadata. - environment: - type: object - data: - type: array - description: >- - An array of at directories in the container the app stores its data - in. Can be empty. Please only list top-level directories. - items: - type: string - user: - type: string - description: The user the container should run as - stop_grace_period: - type: string - description: The grace period for stopping the container. Defaults to 1 minute. - depends_on: - type: array - description: The services the container depends on - entrypoint: - type: - - string - - array - description: The entrypoint for the container - bitcoin_mount_dir: - type: string - description: Where to mount the bitcoin dir - command: - type: - - string - - array - description: The command for the container - init: - type: boolean - description: Whether the container should be run with init - stop_signal: - type: string - description: The signal to send to the container when stopping - noNetwork: - type: boolean - description: >- - Set this to true if the container shouldn't get an IP & port - exposed. - needsHiddenService: - type: boolean - description: >- - Set this to true if the container should be assigned a hidden - service even if it's not the main container. - hiddenServicePort: - type: number - description: >- - Set this to a port if your container exposes multiple ports, but - only one should be a hidden service. - hiddenServicePorts: - type: object - description: >- - Set this to a map of service names to hidden service ports if your - container exposes multiple ports, and all of them should be hidden - services. - patternProperties: - ^[a-zA-Z0-9_]+$: - type: - - number - - array - restart: - type: string - description: When the container should restart. Can be 'always' or 'on-failure'. - additionalProperties: false - required: - - name - - image - additionalProperties: false - -required: - - metadata - - containers - -additionalProperties: false diff --git a/app/lib/composegenerator/shared/env.py b/app/lib/composegenerator/shared/env.py index 69b4318..fb12f1a 100644 --- a/app/lib/composegenerator/shared/env.py +++ b/app/lib/composegenerator/shared/env.py @@ -4,7 +4,7 @@ import re from typing import Union -from lib.composegenerator.v1.types import App +from lib.composegenerator.v2.types import App from lib.composegenerator.shared.const import always_allowed_env from lib.citadelutils import checkArrayContainsAllElements, getEnvVars diff --git a/app/lib/composegenerator/shared/main.py b/app/lib/composegenerator/shared/main.py index 4cc14df..da3c553 100644 --- a/app/lib/composegenerator/shared/main.py +++ b/app/lib/composegenerator/shared/main.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # Main functions -from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container +from lib.composegenerator.v2.types import App, AppStage3, AppStage2, Container from lib.composegenerator.shared.const import permissions diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py new file mode 100644 index 0000000..0b4e47e --- /dev/null +++ b/app/lib/composegenerator/shared/networking.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +import json +from os import path +import random +from lib.composegenerator.v2.types import ContainerStage2, NetworkConfig +from lib.citadelutils import parse_dotenv +from dacite import from_dict + +def getFreePort(networkingFile: str, appId: str): + # Ports used currently in Citadel + usedPorts = [ + # Dashboard + 80, + # Sometimes used by nginx with some setups + 433, + # Dashboard SSL + 443, + # Bitcoin Core P2P + 8333, + # LND gRPC + 10009, + # LND REST + 8080, + # Electrum Server + 50001, + # Tor Proxy + 9050, + ] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + if 'ports' in networkingData: + usedPorts += list(networkingData['ports'].values()) + else: + networkingData['ports'] = {} + + if appId in networkingData['ports']: + return networkingData['ports'][appId] + + while True: + port = str(random.randint(1024, 49151)) + if port not in usedPorts: + # Check if anyhing is listening on the specific port + if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: + networkingData['ports'][appId] = port + break + + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + + return port + +def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1: + raise Exception("Newline in container name") + env_var = "APP_{}_{}_IP".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP + # can be used + usedIps = [] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + + if 'ip_addresses' in networkingData: + usedIps = list(networkingData['ip_addresses'].values()) + else: + networkingData['ip_addresses'] = {} + # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container + # If the IP is already in use, it will be tried again until it's not in use + # If it's not in use, it will be added to the usedIps list and written to the usedIpFile + # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), + # Throw an error, because no more IPs can be used + if len(usedIps) == 235: + raise Exception("No more IPs can be used") + + if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: + ip = networkingData['ip_addresses']["{}-{}".format( + appId, container.name)] + else: + while True: + ip = "10.21.21." + str(random.randint(20, 255)) + if ip not in usedIps: + networkingData['ip_addresses']["{}-{}".format( + appId, container.name)] = ip + break + container.networks = from_dict(data_class=NetworkConfig, data={'default': { + 'ipv4_address': "$" + env_var}}) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): + return container + + # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, ip)) + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + return container + + +def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1 or container.name.find(" ") != -1: + raise Exception("Newline or space in container name") + + env_var = "APP_{}_{}_PORT".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + + port = getFreePort(networkingFile, appId) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(port): + return {"port": port, "env_var": "${{{}}}".format(env_var)} + + # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, port)) + + # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} + # where the outer {{ }} will be replaced by {} in the returned string + return {"port": port, "env_var": "${{{}}}".format(env_var)} + diff --git a/app/lib/composegenerator/v1/generate.py b/app/lib/composegenerator/v1/generate.py deleted file mode 100644 index fa01fd4..0000000 --- a/app/lib/composegenerator/v1/generate.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from lib.composegenerator.v1.types import App, AppStage4, generateApp -from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices -from lib.composegenerator.shared.env import validateEnv -from lib.citadelutils import classToDict -import os - -def createComposeConfigFromV1(app: dict, nodeRoot: str): - envFile = os.path.join(nodeRoot, ".env") - networkingFile = os.path.join(nodeRoot, "apps", "networking.json") - - newApp: App = generateApp(app) - newApp = convertContainerPermissions(newApp) - validateEnv(newApp) - newApp = convertDataDirToVolume(newApp) - newApp = configureIps(newApp, networkingFile, envFile) - newApp = configureMainPort(newApp, nodeRoot) - configureHiddenServices(newApp, nodeRoot) - finalConfig: AppStage4 = convertContainersToServices(newApp) - newApp = classToDict(finalConfig) - del newApp['metadata'] - if "version" in newApp: - del newApp["version"] - # Set version to 3.8 (current compose file version) - newApp = {'version': '3.8', **newApp} - return newApp diff --git a/app/lib/composegenerator/v1/networking.py b/app/lib/composegenerator/v1/networking.py deleted file mode 100644 index 68d1dc1..0000000 --- a/app/lib/composegenerator/v1/networking.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from dacite import from_dict -from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig -from lib.citadelutils import parse_dotenv -import json -from os import path -import random -from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService - - -def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1: - raise Exception("Newline in container name") - env_var = "APP_{}_{}_IP".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP - # can be used - usedIps = [] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - - if 'ip_addresses' in networkingData: - usedIps = list(networkingData['ip_addresses'].values()) - else: - networkingData['ip_addresses'] = {} - # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container - # If the IP is already in use, it will be tried again until it's not in use - # If it's not in use, it will be added to the usedIps list and written to the usedIpFile - # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), - # Throw an error, because no more IPs can be used - if len(usedIps) == 235: - raise Exception("No more IPs can be used") - - if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: - ip = networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] - else: - while True: - ip = "10.21.21." + str(random.randint(20, 255)) - if ip not in usedIps: - networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] = ip - break - container.networks = from_dict(data_class=NetworkConfig, data={'default': { - 'ipv4_address': "$" + env_var}}) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): - return container - - # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, ip)) - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - return container - - -def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1 or container.name.find(" ") != -1: - raise Exception("Newline or space in container name") - - env_var = "APP_{}_{}_PORT".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - - port = getFreePort(networkingFile, appId) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(port): - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, port)) - - # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} - # where the outer {{ }} will be replaced by {} in the returned string - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - -def getMainContainer(app: dict): - if len(app.containers) == 1: - return app.containers[0] - else: - if not app.metadata.mainContainer: - app.metadata.mainContainer = 'main' - for container in app.containers: - if container.name == app.metadata.mainContainer: - return container - raise Exception( - "No main container found for app {}".format(app.metadata.name)) - - -def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: - registryFile = path.join(nodeRoot, "apps", "registry.json") - registry: list = [] - if path.isfile(registryFile): - with open(registryFile, 'r') as f: - registry = json.load(f) - else: - raise Exception("Registry file not found") - - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - - mainContainer = getMainContainer(app) - - portDetails = assignPort(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - containerPort = portDetails['port'] - portAsEnvVar = portDetails['env_var'] - portToAppend = portAsEnvVar - - mainPort = False - - if mainContainer.port: - portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port) - mainPort = mainContainer.port - del mainContainer.port - else: - portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar) - - if mainContainer.ports: - mainContainer.ports.append(portToAppend) - # Set the main port to the first port in the list, if it contains a :, it's the port after the : - # If it doesn't contain a :, it's the port itself - if mainPort == False: - mainPort = mainContainer.ports[0] - if mainPort.find(":") != -1: - mainPort = mainPort.split(":")[1] - else: - mainContainer.ports = [portToAppend] - if mainPort == False: - mainPort = portDetails['port'] - - mainContainer = assignIp(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - - # If the IP wasn't in dotenv before, now it should be - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - - containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace( - "-", "_"), mainContainer.name.upper().replace("-", "_"))] - - hiddenservice = getHiddenService( - app.metadata.name, app.metadata.id, containerIP, mainPort) - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenservice) - - # Also set the port in metadata - app.metadata.port = int(containerPort) - - for registryApp in registry: - if registryApp['id'] == app.metadata.id: - registry[registry.index(registryApp)]['port'] = int(containerPort) - break - - with open(registryFile, 'w') as f: - json.dump(registry, f, indent=4, sort_keys=True) - - return app - - -def configureIps(app: AppStage2, networkingFile: str, envFile: str): - for container in app.containers: - if container.noNetwork: - # Check if port is defined for the container - if container.port: - raise Exception("Port defined for container without network") - if app.metadata.mainContainer == container.name: - raise Exception("Main container without network") - # Skip this iteration of the loop - continue - - container = assignIp(container, app.metadata.id, - networkingFile, envFile) - - return app - - -def configureHiddenServices(app: dict, nodeRoot: str) -> None: - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - hiddenServices = "" - - if len(app.containers) == 1: - mainContainer = app.containers[0] - else: - mainContainer = None - if app.metadata.mainContainer == None: - app.metadata.mainContainer = 'main' - for container in app.containers: - if container.name == app.metadata.mainContainer: - mainContainer = container - break - if mainContainer is None: - raise Exception("No main container found") - - for container in app.containers: - env_var = "APP_{}_{}_IP".format( - app.metadata.id.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - hiddenServices += getContainerHiddenService( - app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name) - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenServices) diff --git a/app/lib/composegenerator/v1/types.py b/app/lib/composegenerator/v1/types.py deleted file mode 100644 index c9556be..0000000 --- a/app/lib/composegenerator/v1/types.py +++ /dev/null @@ -1,151 +0,0 @@ -from typing import Union, List -from dataclasses import dataclass, field -from dacite import from_dict - -@dataclass -class Metadata: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - repo: str - support: str - gallery: List[str] = field(default_factory=list) - dependencies: List[str] = field(default_factory=list) - mainContainer: Union[str, None] = None - updateContainer: Union[str, None] = None - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - -@dataclass -class Container: - name: str - image: str - permissions: list = field(default_factory=list) - ports: list = field(default_factory=list) - port: Union[int, None] = None - environment: Union[dict, None] = None - data: list = field(default_factory=list) - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: list = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - bitcoin_mount_dir: Union[str, None] = None - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - environment_allow: list = field(default_factory=list) - # Only added later - volumes: list = field(default_factory=list) - restart: Union[str, None] = None - -@dataclass -class App: - version: Union[str, int] - metadata: Metadata - containers: List[Container] - -# Generate an app instance from an app dict -def generateApp(appDict): - return from_dict(data_class=App, data=appDict) - -@dataclass -class Network: - ipv4_address: Union[str, None] = None - -@dataclass -class NetworkConfig: - default: Network - -# After converting data dir and defining volumes, stage 2 -@dataclass -class ContainerStage2: - id: str - name: str - image: str - permissions: List[str] = field(default_factory=list) - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - -@dataclass -class AppStage2: - version: Union[str, int] - metadata: Metadata - containers: List[ContainerStage2] - -@dataclass -class MetadataStage3: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - dependencies: List[str] - repo: str - support: str - gallery: List[str] - mainContainer: Union[str, None] = None - updateContainer: Union[str, None] = None - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - -@dataclass -class AppStage3: - version: Union[str, int] - metadata: MetadataStage3 - containers: List[ContainerStage2] - -@dataclass -class ContainerStage4: - id: str - name: str - image: str - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - -@dataclass -class AppStage4: - version: Union[str, int] - metadata: MetadataStage3 - services: List[ContainerStage4] \ No newline at end of file diff --git a/app/lib/composegenerator/v1/utils/networking.py b/app/lib/composegenerator/v1/utils/networking.py deleted file mode 100644 index 2668aa7..0000000 --- a/app/lib/composegenerator/v1/utils/networking.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import json -import os -import random - -from lib.composegenerator.v1.types import Container - -def getFreePort(networkingFile: str, appId: str): - # Ports used currently in Citadel - usedPorts = [ - # Dashboard - 80, - # Sometimes used by nginx with some setups - 433, - # Dashboard SSL - 443, - # Bitcoin Core P2P - 8333, - # LND gRPC - 10009, - # LND REST - 8080, - # Electrum Server - 50001, - # Tor Proxy - 9050, - ] - networkingData = {} - if os.path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - if 'ports' in networkingData: - usedPorts += list(networkingData['ports'].values()) - else: - networkingData['ports'] = {} - - if appId in networkingData['ports']: - return networkingData['ports'][appId] - - while True: - port = str(random.randint(1024, 49151)) - if port not in usedPorts: - # Check if anyhing is listening on the specific port - if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: - networkingData['ports'][appId] = port - break - - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - - return port - - -def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str: - hiddenServices = ''' -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -'''.format(name, id) - for port in ports: - hiddenServices += 'HiddenServicePort {} {}:{}'.format( - port, internalIp, port) - hiddenServices += "\n" - return hiddenServices - - -def getHiddenServiceString(name: str, id: str, internalPort, internalIp: str, publicPort) -> str: - return ''' -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -HiddenServicePort {} {}:{} - -'''.format(name, id, publicPort, internalIp, internalPort) - - -def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str: - return getHiddenServiceString(appName, appId, appPort, appIp, "80") - - -def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str: - if not container.needsHiddenService and not isMainContainer: - return "" - if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer: - print("Container {} for app {} isn't compatible with hidden service assignment".format( - container.name, appName)) - return "" - - if isMainContainer: - if not container.hiddenServicePorts: - return "" - # hiddenServicePorts is a map of hidden service name to port - # We need to generate a hidden service for each one - hiddenServices = "" - for name, port in container.hiddenServicePorts.items(): - if ".." in name: - print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName)) - - # If port is a list, use getHiddenServiceMultiPort - if isinstance(port, list): - hiddenServices += getHiddenServiceMultiPort("{} {}".format(appName, name), "{}-{}".format( - appId, name), containerIp, port) - else: - hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format( - appId, name), port, containerIp, port) - del container.hiddenServicePorts - return hiddenServices - - del container.needsHiddenService - if not container.port: - data = getHiddenServiceString(appName + container.name, "{}-{}".format( - appId, container.name), container.hiddenServicePort, containerIp, "80") - del container.hiddenServicePort - return data - else: - return getHiddenServiceString(appName + container.name, "{}-{}".format( - appId, container.name), container.port, containerIp, container.port) diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py index ec5d4ad..a69540d 100644 --- a/app/lib/composegenerator/v2/networking.py +++ b/app/lib/composegenerator/v2/networking.py @@ -8,7 +8,7 @@ import json from os import path import random from lib.composegenerator.v2.utils.networking import getContainerHiddenService -from lib.composegenerator.v1.networking import assignIp, assignPort +from lib.composegenerator.shared.networking import assignIp, assignPort def getMainContainer(app: App) -> Container: diff --git a/app/lib/composegenerator/v3/networking.py b/app/lib/composegenerator/v3/networking.py index 66d5a96..2e25f55 100644 --- a/app/lib/composegenerator/v3/networking.py +++ b/app/lib/composegenerator/v3/networking.py @@ -7,8 +7,7 @@ from lib.citadelutils import parse_dotenv import json from os import path import random -from lib.composegenerator.v1.networking import assignIp, assignPort - +from lib.composegenerator.shared.networking import assignIp, assignPort def getMainContainerIndex(app: App): if len(app.containers) == 1: @@ -105,4 +104,3 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: with open(envFile, 'a') as f: f.write("{}={}\n".format(portAsEnvVar, app.metadata.port)) return app - diff --git a/app/lib/manage.py b/app/lib/manage.py index 3aabd5e..3033bec 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -25,7 +25,6 @@ except Exception: print("Continuing anyway, but some features won't be available,") print("for example checking for app updates") -from lib.composegenerator.v1.generate import createComposeConfigFromV1 from lib.composegenerator.v2.generate import createComposeConfigFromV2 from lib.composegenerator.v3.generate import createComposeConfigFromV3 from lib.validate import findAndValidateApps @@ -188,10 +187,7 @@ def getApp(appFile: str, appId: str): raise Exception("Error: Could not find metadata in " + appFile) app["metadata"]["id"] = appId - if 'version' in app and str(app['version']) == "1": - print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId)) - return createComposeConfigFromV1(app, nodeRoot) - elif 'version' in app and str(app['version']) == "2": + if 'version' in app and str(app['version']) == "2": print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId)) return createComposeConfigFromV2(app, nodeRoot) elif 'version' in app and str(app['version']) == "3": diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 6db5672..4ab7ca1 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -7,7 +7,7 @@ import yaml from lib.composegenerator.next.stage1 import createCleanConfigFromV3 from lib.composegenerator.v2.networking import getMainContainer -from lib.composegenerator.v1.networking import getFreePort +from lib.composegenerator.shared.networking import getFreePort from lib.entropy import deriveEntropy from typing import List import json diff --git a/app/lib/validate.py b/app/lib/validate.py index 9dfe236..16009d6 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -9,8 +9,6 @@ import yaml scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") -with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f: - schemaVersion1 = yaml.safe_load(f) with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: schemaVersion2 = yaml.safe_load(f) with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: @@ -19,15 +17,7 @@ with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: # Validates app data # Returns true if valid, false otherwise def validateApp(app: dict): - if 'version' in app and str(app['version']) == "1": - try: - validate(app, schemaVersion1) - return True - # Catch and log any errors, and return false - except Exception as e: - print(e) - return False - elif 'version' in app and str(app['version']) == "2": + if 'version' in app and str(app['version']) == "2": try: validate(app, schemaVersion2) return True diff --git a/docker-compose.yml b/docker-compose.yml index 60eb321..a83901b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -234,6 +234,13 @@ services: networks: default: ipv4_address: $REDIS_IP + + app-cli: + container_name: app-cli + image: ghcr.io/runcitadel/app-cli:main@sha256:694e52fa9da1ac976165f269c17e27803032a05a76293dfe3589a50813306ded + volumes: + - ${PWD}/apps:/apps + networks: default: name: citadel_main_network