diff --git a/app/app-standard-v2.yml b/app/app-standard-v2.yml deleted file mode 100644 index c630d92..0000000 --- a/app/app-standard-v2.yml +++ /dev/null @@ -1,201 +0,0 @@ -# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema -$schema: https://json-schema.org/draft/2020-12/schema - - -title: Citadel app.yml v2 -description: The second revision of Citadel's app.yml format -type: object - -properties: - version: - type: - - string - - number - description: The version of the app.yml format you're using. - - metadata: - type: object - properties: - name: - description: Displayed name of the app - type: string - version: - description: Displayed version for the app - type: string - category: - description: The category you'd put the app in - type: string - tagline: - description: A clever tagline - type: string - description: - description: A longer description of the app - type: string - developer: - description: The awesome people behind the app - type: string - website: - description: Displayed version for the app - type: string - dependencies: - description: The services the app depends on - type: array - items: - type: string - repo: - description: The development repository for your app - type: string - support: - description: A link to the app support wiki/chat/... - type: string - gallery: - type: array - description: >- - URLs or paths in the runcitadel/app-images/[app-name] folder with app - images - items: - type: string - path: - description: The path of the app's visible site the open button should open - type: string - defaultPassword: - description: The app's default password Set this to $APP_SEED if the password is the environment variable $APP_SEED. - type: string - torOnly: - description: Whether the app is only available over tor - type: boolean - updateContainer: - type: - - string - - array - description: The container(s) the developer system should automatically update. - lightningImplementation: - description: The supported lightning implementation for this app. If your app supports multiple, please publish a separate app.yml for each implementation. - type: string - enum: - - lnd - - c-lightning - required: - - name - - version - - category - - tagline - - description - - developer - - website - - repo - - support - - gallery - additionalProperties: false - - containers: - type: array - items: - type: object - properties: - name: - type: string - image: - type: string - permissions: - type: array - items: - type: string - enum: - - lnd - - c-lightning - - bitcoind - - electrum - - root - - hw - ports: - type: array - items: - type: - - string - - number - port: - type: number - description: >- - If this is the main container, the port inside the container which - will be exposed to the outside as the port specified in metadata. - If this is not set, the port is passed as an env variable in the format APP_${APP_NAME}_${CONTAINER_NAME}_PORT - environment: - type: object - data: - type: array - description: >- - An array of at directories in the container the app stores its data - in. Can be empty. Please only list top-level directories. - items: - type: string - user: - type: string - description: The user the container should run as - stop_grace_period: - type: string - description: The grace period for stopping the container. Defaults to 1 minute. - depends_on: - type: array - description: The services the container depends on - entrypoint: - type: - - string - - array - description: The entrypoint for the container - bitcoin_mount_dir: - type: string - description: Where to mount the bitcoin dir - lnd_mount_dir: - type: string - description: Where to mount the lnd dir - c_lightning_mount_dir: - type: string - description: Where to mount the c-lightning dir - command: - type: - - string - - array - description: The command for the container - init: - type: boolean - description: Whether the container should be run with init - stop_signal: - type: string - description: The signal to send to the container when stopping - noNetwork: - type: boolean - description: >- - Set this to true if the container shouldn't get an IP & port - exposed. This isn't necessary, but helps the docker-compose.yml generator to generate a cleaner output. - hiddenServicePorts: - type: - - object - - number - - array - items: - type: - - string - - number - - array - description: >- - This can either be a map of hidden service names (human readable names, not the .onion URL, and strings, not numbers) - to a port if your app needs multiple hidden services on different ports, - a map of port inside to port on the hidden service (if your app has multiple ports on one hidden service), - or simply one port number if your apps hidden service should only expose one port to the outside which isn't 80. - restart: - type: string - description: When the container should restart. Can be 'always' or 'on-failure'. - network_mode: - type: string - additionalProperties: false - required: - - name - - image - additionalProperties: false - -required: - - metadata - - containers - -additionalProperties: false diff --git a/app/lib/composegenerator/shared/const.py b/app/lib/composegenerator/shared/const.py deleted file mode 100644 index bb7d1ea..0000000 --- a/app/lib/composegenerator/shared/const.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -def permissions(): - return { - "lnd": { - "environment_allow": [ - "LND_IP", - "LND_GRPC_PORT", - "LND_REST_PORT", - "BITCOIN_NETWORK" - ], - "volumes": [ - '${LND_DATA_DIR}:/lnd:ro' - ] - }, - "bitcoind": { - "environment_allow": [ - "BITCOIN_IP", - "BITCOIN_NETWORK", - "BITCOIN_P2P_PORT", - "BITCOIN_RPC_PORT", - "BITCOIN_RPC_USER", - "BITCOIN_RPC_PASS", - "BITCOIN_RPC_AUTH", - "BITCOIN_ZMQ_RAWBLOCK_PORT", - "BITCOIN_ZMQ_RAWTX_PORT", - "BITCOIN_ZMQ_HASHBLOCK_PORT", - "BITCOIN_ZMQ_SEQUENCE_PORT", - ], - "volumes": [ - "${BITCOIN_DATA_DIR}:/bitcoin" - ] - }, - "electrum": { - "environment_allow": [ - "ELECTRUM_IP", - "ELECTRUM_PORT", - ], - "volumes": [] - }, - "c-lightning": { - "environment_allow": [ - "C_LIGHTNING_IP" - ], - "volumes": [] - }, - } - -# Vars which are always allowed without permissions -always_allowed_env = ["TOR_PROXY_IP", "TOR_PROXY_PORT", - "APP_DOMAIN", "APP_HIDDEN_SERVICE", "BITCOIN_NETWORK"] diff --git a/app/lib/composegenerator/shared/env.py b/app/lib/composegenerator/shared/env.py deleted file mode 100644 index fb12f1a..0000000 --- a/app/lib/composegenerator/shared/env.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import re -from typing import Union -from lib.composegenerator.v2.types import App -from lib.composegenerator.shared.const import always_allowed_env -from lib.citadelutils import checkArrayContainsAllElements, getEnvVars - -def validateEnvByValue(env: list, allowed: list, app_name: str): - # Combine always_allowed_env with allowed into one list - # Then check if all elements in env are in the resulting list - all_allowed = allowed + always_allowed_env - if not checkArrayContainsAllElements(env, all_allowed): - # This has a weird syntax, and it confuses VSCode, but it works - validation_regex = r"APP_{}(\S+)".format( - app_name.upper().replace("-", "_")) - for key in env: - # If the key is neither in all_allowed nor is a full match against the validation regex, print a warning and return false - if key not in all_allowed and re.fullmatch(validation_regex, key) is None and not key.startswith("APP_HIDDEN_SERVICE")and not key.startswith("APP_SEED"): - print("Invalid environment variable {} in app {}".format( - key, app_name)) - return False - return True - -def validateEnvStringOrListorDict(env: Union[str, Union[list, dict]], existingEnv: list, app_name: str, container_name: str): - envList = [] - if isinstance(env, dict): - envList = env.values() - elif isinstance(env, list): - envList = env - elif isinstance(env, str): - envList = [env] - for envVar in envList: - if not validateEnvByValue(getEnvVars(envVar), existingEnv, app_name): - raise Exception("Env var {} not defined for container {} of app {}".format(envVar, container_name, app_name)) - - -def validateEnv(app: App): - # For every container of the app, check if all env vars in the strings in environment are defined in env - for container in app.containers: - if container is not None: - if container.environment_allow: - existingEnv = container.environment_allow - del container.environment_allow - else: - existingEnv = [] - if container.environment: - validateEnvStringOrListorDict(container.command, existingEnv, app.metadata.id, container.name) - validateEnvStringOrListorDict(container.entrypoint, existingEnv, app.metadata.id, container.name) - validateEnvStringOrListorDict(container.environment, existingEnv, app.metadata.id, container.name) - return app diff --git a/app/lib/composegenerator/shared/main.py b/app/lib/composegenerator/shared/main.py deleted file mode 100644 index ff9922d..0000000 --- a/app/lib/composegenerator/shared/main.py +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -# Main functions -from lib.composegenerator.v2.types import App, AppStage3 -from lib.composegenerator.shared.const import permissions - - -def convertContainerPermissions(app: App) -> App: - for container in app.containers: - for permission in container.permissions: - if permission in permissions(): - container.environment_allow.extend(permissions()[permission]['environment_allow']) - container.volumes.extend(permissions()[permission]['volumes']) - else: - print("Warning: container {} of app {} defines unknown permission {}".format(container.name, app.metadata.name, permission)) - return app - -def convertContainersToServices(app: AppStage3) -> AppStage3: - services = {} - for container in app.containers: - if container.permissions: - del container.permissions - services[container.name] = container - del services[container.name].name - del app.containers - app.services = services - return app diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py deleted file mode 100644 index bf39041..0000000 --- a/app/lib/composegenerator/shared/networking.py +++ /dev/null @@ -1,174 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import json -from os import path -import random -from lib.composegenerator.shared.utils.networking import getContainerHiddenService -from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container -from lib.citadelutils import parse_dotenv -from dacite import from_dict - -def getMainContainer(app: App) -> Container: - if len(app.containers) == 1: - return app.containers[0] - else: - for container in app.containers: - # Main is recommended, support web for easier porting from Umbrel - if container.name == 'main' or container.name == 'web': - return container - # Fallback to first container - return app.containers[0] - -def assignIpV4(appId: str, containerName: str): - scriptDir = path.dirname(path.realpath(__file__)) - nodeRoot = path.join(scriptDir, "..", "..", "..", "..") - networkingFile = path.join(nodeRoot, "apps", "networking.json") - envFile = path.join(nodeRoot, ".env") - cleanContainerName = containerName.strip() - # If the name still contains a newline, throw an error - if cleanContainerName.find("\n") != -1: - raise Exception("Newline in container name") - env_var = "APP_{}_{}_IP".format( - appId.upper().replace("-", "_"), - cleanContainerName.upper().replace("-", "_") - ) - # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP - # can be used - usedIps = [] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - - if 'ip_addresses' in networkingData: - usedIps = list(networkingData['ip_addresses'].values()) - else: - networkingData['ip_addresses'] = {} - # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container - # If the IP is already in use, it will be tried again until it's not in use - # If it's not in use, it will be added to the usedIps list and written to the usedIpFile - # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), - # Throw an error, because no more IPs can be used - if len(usedIps) == 235: - raise Exception("No more IPs can be used") - - if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']: - ip = networkingData['ip_addresses']["{}-{}".format( - appId, cleanContainerName)] - else: - while True: - ip = "10.21.21." + str(random.randint(20, 255)) - if ip not in usedIps: - networkingData['ip_addresses']["{}-{}".format( - appId, cleanContainerName)] = ip - break - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): - return - - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, ip)) - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - -def assignIp(container: ContainerStage2, appId: str) -> ContainerStage2: - scriptDir = path.dirname(path.realpath(__file__)) - nodeRoot = path.join(scriptDir, "..", "..", "..", "..") - networkingFile = path.join(nodeRoot, "apps", "networking.json") - envFile = path.join(nodeRoot, ".env") - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1: - raise Exception("Newline in container name") - env_var = "APP_{}_{}_IP".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP - # can be used - usedIps = [] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - - if 'ip_addresses' in networkingData: - usedIps = list(networkingData['ip_addresses'].values()) - else: - networkingData['ip_addresses'] = {} - # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container - # If the IP is already in use, it will be tried again until it's not in use - # If it's not in use, it will be added to the usedIps list and written to the usedIpFile - # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), - # Throw an error, because no more IPs can be used - if len(usedIps) == 235: - raise Exception("No more IPs can be used") - - if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: - ip = networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] - else: - while True: - ip = "10.21.21." + str(random.randint(20, 255)) - if ip not in usedIps: - networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] = ip - break - container.networks = from_dict(data_class=NetworkConfig, data={'default': { - 'ipv4_address': "$" + env_var}}) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): - return container - - # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, ip)) - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - return container - -def configureIps(app: AppStage2, networkingFile: str, envFile: str): - for container in app.containers: - if container.network_mode and container.network_mode == "host": - continue - if container.noNetwork: - # Check if port is defined for the container - if container.port: - raise Exception("Port defined for container without network") - if getMainContainer(app).name == container.name: - raise Exception("Main container without network") - # Skip this iteration of the loop - continue - - container = assignIp(container, app.metadata.id) - - return app - -def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3: - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - hiddenServices = "" - - mainContainer = getMainContainer(app) - - for container in app.containers: - if container.network_mode and container.network_mode == "host": - continue - env_var = "APP_{}_{}_IP".format( - app.metadata.id.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - hiddenServices += getContainerHiddenService( - app.metadata, container, dotEnv[env_var], container.name == mainContainer.name) - if container.hiddenServicePorts: - del container.hiddenServicePorts - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenServices) - return app diff --git a/app/lib/composegenerator/shared/utils/networking.py b/app/lib/composegenerator/shared/utils/networking.py deleted file mode 100644 index e27ef16..0000000 --- a/app/lib/composegenerator/shared/utils/networking.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from lib.composegenerator.v2.types import Metadata, Container - - -def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str: - hiddenServices = """ -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -""".format( - name, id - ) - for port in ports: - hiddenServices += "HiddenServicePort {} {}:{}".format(port, internalIp, port) - hiddenServices += "\n" - return hiddenServices - - -def getHiddenServiceString( - name: str, id: str, internalPort, internalIp: str, publicPort -) -> str: - return """ -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -HiddenServicePort {} {}:{} - -""".format( - name, id, publicPort, internalIp, internalPort - ) - - -def getContainerHiddenService( - metadata: Metadata, container: Container, containerIp: str, isMainContainer: bool -) -> str: - if isMainContainer and not container.hiddenServicePorts: - return getHiddenServiceString( - metadata.name, metadata.id, metadata.internalPort, containerIp, 80 - ) - - if container.hiddenServicePorts: - if isinstance(container.hiddenServicePorts, int): - return getHiddenServiceString( - "{} {}".format(metadata.name, container.name), - metadata.id if isMainContainer else "{}-{}".format(metadata.id, container.name), - container.hiddenServicePorts, - containerIp, - container.hiddenServicePorts, - ) - elif isinstance(container.hiddenServicePorts, list): - return getHiddenServiceMultiPort( - "{} {}".format(metadata.name, container.name), - metadata.id if isMainContainer else "{}-{}".format(metadata.id, container.name), - containerIp, - container.hiddenServicePorts, - ) - elif isinstance(container.hiddenServicePorts, dict): - additionalHiddenServices = {} - hiddenServices = "# {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format( - metadata.name, container.name, metadata.id, container.name - ) - initialHiddenServices = "# {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format( - metadata.name, container.name, metadata.id, container.name - ) - otherHiddenServices = "" - for key, value in container.hiddenServicePorts.items(): - if isinstance(key, int): - hiddenServices += "HiddenServicePort {} {}:{}".format( - key, containerIp, value - ) - hiddenServices += "\n" - else: - additionalHiddenServices[key] = value - for key, value in additionalHiddenServices.items(): - if isinstance(value, int): - otherHiddenServices += "# {} {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format( - metadata.name, container.name, key, metadata.id, key - ) - otherHiddenServices += "HiddenServicePort {} {}:{}".format( - value, containerIp, value - ) - otherHiddenServices += "\n" - elif isinstance(value, list): - otherHiddenServices += getHiddenServiceMultiPort( - "{} {}".format(metadata.name, key), "{}-{}".format(metadata.id, key), containerIp, value - ) - - if hiddenServices == initialHiddenServices: - return otherHiddenServices - else : - return hiddenServices + "\n" + otherHiddenServices - del container.hiddenServicePorts - - return "" diff --git a/app/lib/composegenerator/v2/generate.py b/app/lib/composegenerator/v2/generate.py deleted file mode 100644 index b236c2a..0000000 --- a/app/lib/composegenerator/v2/generate.py +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp -from lib.composegenerator.v2.networking import configureMainPort -from lib.composegenerator.shared.networking import configureHiddenServices, configureIps -from lib.composegenerator.shared.main import convertContainerPermissions, convertContainersToServices -from lib.composegenerator.shared.env import validateEnv -from lib.citadelutils import classToDict -import os - -def convertDataDirToVolume(app: App) -> AppStage2: - for container in app.containers: - # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes - # Also, a datadir shouldn't start with a / - for dataDir in container.data: - if dataDir.find("..") == -1 and dataDir[0] != "/": - container.volumes.append( - '${APP_DATA_DIR}/' + dataDir) - else: - print("Data dir " + dataDir + - " contains invalid characters") - del container.data - if container.bitcoin_mount_dir != None: - if not 'bitcoind' in container.permissions: - print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name)) - # Skip this container - continue - # Also skip the container if container.bitcoin_mount_dir contains a : - if container.bitcoin_mount_dir.find(":") == -1: - container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir) - del container.bitcoin_mount_dir - if container.lnd_mount_dir != None: - if not 'lnd' in container.permissions: - print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name)) - # Skip this container - continue - # Also skip the container if container.lnd_mount_dir contains a : - if container.lnd_mount_dir.find(":") == -1: - container.volumes.append('${LND_DATA_DIR}:' + container.lnd_mount_dir) - del container.lnd_mount_dir - if container.c_lightning_mount_dir != None: - if not 'lnd' in container.permissions: - print("Warning: container {} of app {} defines c_lightning_mount_dir but doesn't request c-lightning permission".format(container.name, app.metadata.name)) - # Skip this container - continue - # Also skip the container if container.c_lightning.mount_dir contains a : - if container.c_lightning_mount_dir.find(":") == -1: - container.volumes.append('${C_LIGHTNING_DATA_DIR}:' + container.c_lightning_mount_dir) - del container.c_lightning_mount_dir - - return app - -def createComposeConfigFromV2(app: dict, nodeRoot: str): - envFile = os.path.join(nodeRoot, ".env") - networkingFile = os.path.join(nodeRoot, "apps", "networking.json") - - newApp: App = generateApp(app) - newApp = convertContainerPermissions(newApp) - newApp = validateEnv(newApp) - newApp = convertDataDirToVolume(newApp) - newApp = configureIps(newApp, networkingFile, envFile) - newApp = configureMainPort(newApp, nodeRoot) - newApp = configureHiddenServices(newApp, nodeRoot) - finalConfig: AppStage4 = convertContainersToServices(newApp) - newApp = classToDict(finalConfig) - del newApp['metadata'] - if "version" in newApp: - del newApp["version"] - return newApp diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py deleted file mode 100644 index e48d37e..0000000 --- a/app/lib/composegenerator/v2/networking.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from lib.citadelutils import parse_dotenv -from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container -import json -from os import path -import os -import random -from lib.composegenerator.shared.networking import assignIp, getMainContainer -from lib.citadelutils import FileLock - -def getFreePort(networkingFile: str, appId: str): - # Ports used currently in Citadel - usedPorts = [ - # Dashboard - 80, - # Sometimes used by nginx with some setups - 433, - # Dashboard SSL - 443, - # Bitcoin Core P2P - 8333, - # LND gRPC - 10009, - # LND REST - 8080, - # Electrum Server - 50001, - # Tor Proxy - 9050, - ] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - if 'ports' in networkingData: - usedPorts += list(networkingData['ports'].values()) - else: - networkingData['ports'] = {} - - if appId in networkingData['ports']: - return networkingData['ports'][appId] - - while True: - port = str(random.randint(1024, 49151)) - if port not in usedPorts: - # Check if anyhing is listening on the specific port - if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: - networkingData['ports'][appId] = port - break - - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - - return port - -def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1 or container.name.find(" ") != -1: - raise Exception("Newline or space in container name") - - env_var = "APP_{}_{}_PORT".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - - port = getFreePort(networkingFile, appId) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(port): - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, port)) - - # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} - # where the outer {{ }} will be replaced by {} in the returned string - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - -def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: - lock = FileLock("citadel_registry_lock", dir="/tmp") - lock.acquire() - registryFile = path.join(nodeRoot, "apps", "registry.json") - registry: list = [] - if path.isfile(registryFile): - with open(registryFile, 'r') as f: - registry = json.load(f) - else: - raise Exception("Registry file not found") - - mainContainer = getMainContainer(app) - - portDetails = assignPort(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - containerPort = portDetails['port'] - portAsEnvVar = portDetails['env_var'] - portToAppend = portAsEnvVar - - mainPort = False - - if mainContainer.port: - portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port) - mainPort = mainContainer.port - del mainContainer.port - else: - portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar) - - if mainContainer.ports: - mainContainer.ports.append(portToAppend) - # Set the main port to the first port in the list, if it contains a :, it's the port after the : - # If it doesn't contain a :, it's the port itself - if mainPort == False: - mainPort = mainContainer.ports[0] - if mainPort.find(":") != -1: - mainPort = mainPort.split(":")[1] - else: - mainContainer.ports = [portToAppend] - if mainPort == False: - mainPort = portDetails['port'] - - if mainContainer.network_mode != "host": - mainContainer = assignIp(mainContainer, app.metadata.id) - - # Also set the port in metadata - app.metadata.port = int(containerPort) - if mainPort: - app.metadata.internalPort = int(mainPort) - else: - app.metadata.internalPort = int(containerPort) - - for registryApp in registry: - if registryApp['id'] == app.metadata.id: - registry[registry.index(registryApp)]['port'] = int(containerPort) - registry[registry.index(registryApp)]['internalPort'] = app.metadata.internalPort - break - - with open(registryFile, 'w') as f: - json.dump(registry, f, indent=4, sort_keys=True) - lock.release() - return app diff --git a/app/lib/composegenerator/v2/types.py b/app/lib/composegenerator/v2/types.py deleted file mode 100644 index 93bbcf3..0000000 --- a/app/lib/composegenerator/v2/types.py +++ /dev/null @@ -1,156 +0,0 @@ -from typing import Union, List -from dataclasses import dataclass, field -from dacite import from_dict - -@dataclass -class Metadata: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - repo: str - support: str - gallery: List[str] = field(default_factory=list) - dependencies: List[str] = field(default_factory=list) - updateContainer: Union[str, Union[list, None]] = field(default_factory=list) - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - lightningImplementation: Union[str, None] = None - # Added automatically later - port: int = 0 - internalPort: int = 0 - -@dataclass -class Container: - name: str - image: str - permissions: list = field(default_factory=list) - ports: list = field(default_factory=list) - port: Union[int, None] = None - environment: Union[dict, None] = None - data: list = field(default_factory=list) - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: list = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - bitcoin_mount_dir: Union[str, None] = None - lnd_mount_dir: Union[str, None] = None - c_lightning_mount_dir: Union[str, None] = None - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list) - environment_allow: list = field(default_factory=list) - network_mode: Union[str, None] = None - # Only added later - volumes: list = field(default_factory=list) - restart: Union[str, None] = None - -@dataclass -class App: - version: Union[str, int] - metadata: Metadata - containers: List[Container] - -# Generate an app instance from an app dict -def generateApp(appDict): - return from_dict(data_class=App, data=appDict) - -@dataclass -class Network: - ipv4_address: Union[str, None] = None - -@dataclass -class NetworkConfig: - default: Network - -# After converting data dir and defining volumes, stage 2 -@dataclass -class ContainerStage2: - id: str - name: str - image: str - permissions: List[str] = field(default_factory=list) - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list) - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - network_mode: Union[str, None] = None - -@dataclass -class AppStage2: - version: Union[str, int] - metadata: Metadata - containers: List[ContainerStage2] - -@dataclass -class MetadataStage3: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - dependencies: List[str] - repo: str - support: str - gallery: List[str] - updateContainer: Union[str, Union[list, None]] = field(default_factory=list) - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - lightningImplementation: Union[str, None] = None - # Added automatically later - port: int = 0 - internalPort: int = 0 - -@dataclass -class AppStage3: - version: Union[str, int] - metadata: MetadataStage3 - containers: List[ContainerStage2] - -@dataclass -class ContainerStage4: - id: str - name: str - image: str - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list) - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - network_mode: Union[str, None] = None - -@dataclass -class AppStage4: - version: Union[str, int] - metadata: MetadataStage3 - services: List[ContainerStage4] \ No newline at end of file diff --git a/app/lib/manage.py b/app/lib/manage.py index 9996d09..d09b426 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -30,7 +30,6 @@ except Exception: print("Continuing anyway, but some features won't be available,") print("for example checking for app updates") -from lib.composegenerator.v2.generate import createComposeConfigFromV2 from lib.validate import findAndValidateApps from lib.metadata import getAppRegistry from lib.entropy import deriveEntropy @@ -172,12 +171,7 @@ def update(verbose: bool = False): thread.start() threads.append(thread) else: - appCompose = getApp(appDefinition, app) - with open(composeFile, "w") as f: - if appCompose: - f.write(yaml.dump(appCompose, sort_keys=False)) - if verbose: - print("Wrote " + app + " to " + composeFile) + raise Exception("Error: Unsupported version of app.yml") except Exception as err: print("Failed to convert app {}".format(app)) print(traceback.format_exc()) @@ -241,19 +235,6 @@ def stopInstalled(): threads.append(thread) joinThreads(threads) -# Loads an app.yml and converts it to a docker-compose.yml -def getApp(app, appId: str): - if not "metadata" in app: - raise Exception("Error: Could not find metadata in " + appId) - app["metadata"]["id"] = appId - - if 'version' in app and str(app['version']) == "2": - print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId)) - return createComposeConfigFromV2(app, nodeRoot) - else: - raise Exception("Error: Unsupported version of app.yml") - - def compose(app, arguments): if not os.path.isdir(os.path.join(appsDir, app)): print("Warning: App {} doesn't exist on Citadel".format(app)) diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 093a496..3d75ece 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -6,8 +6,7 @@ import os import yaml import traceback -from lib.composegenerator.shared.networking import assignIp, assignIpV4 -from lib.composegenerator.v2.types import Container +from lib.citadelutils import parse_dotenv from lib.entropy import deriveEntropy from dacite import from_dict @@ -15,6 +14,59 @@ appPorts = {} appPortMap = {} disabledApps = [] +def assignIpV4(appId: str, containerName: str): + scriptDir = path.dirname(path.realpath(__file__)) + nodeRoot = path.join(scriptDir, "..", "..", "..", "..") + networkingFile = path.join(nodeRoot, "apps", "networking.json") + envFile = path.join(nodeRoot, ".env") + cleanContainerName = containerName.strip() + # If the name still contains a newline, throw an error + if cleanContainerName.find("\n") != -1: + raise Exception("Newline in container name") + env_var = "APP_{}_{}_IP".format( + appId.upper().replace("-", "_"), + cleanContainerName.upper().replace("-", "_") + ) + # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP + # can be used + usedIps = [] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + + if 'ip_addresses' in networkingData: + usedIps = list(networkingData['ip_addresses'].values()) + else: + networkingData['ip_addresses'] = {} + # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container + # If the IP is already in use, it will be tried again until it's not in use + # If it's not in use, it will be added to the usedIps list and written to the usedIpFile + # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), + # Throw an error, because no more IPs can be used + if len(usedIps) == 235: + raise Exception("No more IPs can be used") + + if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']: + ip = networkingData['ip_addresses']["{}-{}".format( + appId, cleanContainerName)] + else: + while True: + ip = "10.21.21." + str(random.randint(20, 255)) + if ip not in usedIps: + networkingData['ip_addresses']["{}-{}".format( + appId, cleanContainerName)] = ip + break + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): + return + + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, ip)) + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + def appPortsToMap(): for port in appPorts: appId = appPorts[port]["app"] @@ -64,9 +116,7 @@ def getAppRegistry(apps, app_path, portCache): virtual_apps[implements] = [] virtual_apps[implements].append(app) app_metadata.append(metadata) - if version == 2: - getPortsV2App(app_yml, app) - elif version == 3: + if version == 3: getPortsV3App(app_yml, app) elif version == 4: getPortsV4App(app_yml, app) @@ -150,16 +200,6 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna "dynamic": isDynamic, } -def getPortsV2App(app, appId): - for appContainer in app["containers"]: - if "port" in appContainer: - validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0) - if "ports" in appContainer: - for port in appContainer["ports"]: - realPort = int(str(port).split(":")[0]) - validatePort(appContainer["name"], appContainer, realPort, appId, 2) - - def getPortsV3App(app, appId): for appContainer in app["containers"]: containerAsDataClass = from_dict(data_class=Container, data=appContainer) diff --git a/app/lib/validate.py b/app/lib/validate.py index 19fbceb..e2c7a55 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -11,27 +11,16 @@ import traceback scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") nodeRoot = os.path.join(scriptDir, "..") -with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: - schemaVersion2 = yaml.safe_load(f) - with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file: dependencies = yaml.safe_load(file) # Validates app data # Returns true if valid, false otherwise def validateApp(app: dict): - if 'version' in app and str(app['version']) == "2": - try: - validate(app, schemaVersion2) - return True - # Catch and log any errors, and return false - except Exception as e: - print(traceback.format_exc()) - return False - elif 'version' in app and str(app['version']) == "3": + if 'version' in app and str(app['version']) == "3": # The app-cli does this validation now return True - elif 'version' not in app and 'citadel_version' not in app: + elif 'citadel_version' not in app: print("Unsupported app version") return False else: