diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..de288e1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.formatting.provider": "black" +} \ No newline at end of file diff --git a/app/app-standard-v2.yml b/app/app-standard-v2.yml new file mode 100644 index 0000000..fdc66b1 --- /dev/null +++ b/app/app-standard-v2.yml @@ -0,0 +1,199 @@ +# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema +$schema: https://json-schema.org/draft/2020-12/schema + + +title: Citadel app.yml v2 +description: The second revision of Citadel's app.yml format +type: object + +properties: + version: + type: + - string + - number + description: The version of the app.yml format you're using. + + metadata: + type: object + properties: + name: + description: Displayed name of the app + type: string + version: + description: Displayed version for the app + type: string + category: + description: The category you'd put the app in + type: string + tagline: + description: A clever tagline + type: string + description: + description: A longer description of the app + type: string + developer: + description: The awesome people behind the app + type: string + website: + description: Displayed version for the app + type: string + dependencies: + description: The services the app depends on + type: array + items: + type: string + repo: + description: The development repository for your app + type: string + support: + description: A link to the app support wiki/chat/... + type: string + gallery: + type: array + description: >- + URLs or paths in the runcitadel/app-images/[app-name] folder with app + images + items: + type: string + path: + description: The path of the app's visible site the open button should open + type: string + defaultPassword: + description: The app's default password Set this to $APP_SEED if the password is the environment variable $APP_SEED. + type: string + torOnly: + description: Whether the app is only available over tor + type: boolean + updateContainer: + type: + - string + - array + description: The container(s) the developer system should automatically update. + lightningImplementation: + description: The supported lightning implementation for this app. If your app supports multiple, please publish a separate app.yml for each implementation. + type: string + enum: + - lnd + - c-lightning + required: + - name + - version + - category + - tagline + - description + - developer + - website + - repo + - support + - gallery + additionalProperties: false + + containers: + type: array + items: + type: object + properties: + name: + type: string + image: + type: string + permissions: + type: array + items: + type: string + enum: + - lnd + - c-lightning + - bitcoind + - electrum + - root + - hw + ports: + type: array + items: + type: + - string + - number + port: + type: number + description: >- + If this is the main container, the port inside the container which + will be exposed to the outside as the port specified in metadata. + If this is not set, the port is passed as an env variable in the format APP_${APP_NAME}_${CONTAINER_NAME}_PORT + environment: + type: object + data: + type: array + description: >- + An array of at directories in the container the app stores its data + in. Can be empty. Please only list top-level directories. + items: + type: string + user: + type: string + description: The user the container should run as + stop_grace_period: + type: string + description: The grace period for stopping the container. Defaults to 1 minute. + depends_on: + type: array + description: The services the container depends on + entrypoint: + type: + - string + - array + description: The entrypoint for the container + bitcoin_mount_dir: + type: string + description: Where to mount the bitcoin dir + lnd_mount_dir: + type: string + description: Where to mount the lnd dir + c_lightning_mount_dir: + type: string + description: Where to mount the c-lightning dir + command: + type: + - string + - array + description: The command for the container + init: + type: boolean + description: Whether the container should be run with init + stop_signal: + type: string + description: The signal to send to the container when stopping + noNetwork: + type: boolean + description: >- + Set this to true if the container shouldn't get an IP & port + exposed. This isn't necessary, but helps the docker-compose.yml generator to generate a cleaner output. + hiddenServicePorts: + type: + - object + - number + - array + items: + type: + - string + - number + - array + description: >- + This can either be a map of hidden service names (human readable names, not the .onion URL, and strings, not numbers) + to a port if your app needs multiple hidden services on different ports, + a map of port inside to port on the hidden service (if your app has multiple ports on one hidden service), + or simply one port number if your apps hidden service should only expose one port to the outside which isn't 80. + restart: + type: string + description: When the container should restart. Can be 'always' or 'on-failure'. + additionalProperties: false + required: + - name + - image + additionalProperties: false + +required: + - metadata + - containers + +additionalProperties: false diff --git a/app/lib/composegenerator/v1/generate.py b/app/lib/composegenerator/v1/generate.py index 7cd9336..fa01fd4 100644 --- a/app/lib/composegenerator/v1/generate.py +++ b/app/lib/composegenerator/v1/generate.py @@ -10,10 +10,6 @@ from lib.citadelutils import classToDict import os def createComposeConfigFromV1(app: dict, nodeRoot: str): - if "version" in app: - if str(app['version']) != "1": - print("Warning: app version is not supported") - return False envFile = os.path.join(nodeRoot, ".env") networkingFile = os.path.join(nodeRoot, "apps", "networking.json") diff --git a/app/lib/composegenerator/v2/generate.py b/app/lib/composegenerator/v2/generate.py new file mode 100644 index 0000000..a159568 --- /dev/null +++ b/app/lib/composegenerator/v2/generate.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp +from lib.composegenerator.v2.networking import configureHiddenServices, configureIps, configureMainPort +from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices +from lib.composegenerator.shared.env import validateEnv +from lib.citadelutils import classToDict +import os + +def convertDataDirToVolumeGen2(app: App) -> AppStage2: + app = convertDataDirToVolume(app) + for container in app.containers: + if container.lnd_mount_dir != None: + if not 'lnd' in container.permissions: + print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name)) + # Skip this container + continue + # Also skip the container if container.lnd_mount_dir contains a : + if container.lnd_mount_dir.find(":") == -1: + container.volumes.append('${LND_DATA_DIR}:' + container.lnd_mount_dir) + del container.lnd_mount_dir + if container.c_lightning_mount_dir != None: + if not 'lnd' in container.permissions: + print("Warning: container {} of app {} defines c_lightning_mount_dir but doesn't request c-lightning permission".format(container.name, app.metadata.name)) + # Skip this container + continue + # Also skip the container if container.c_lightning.mount_dir contains a : + if container.c_lightning_mount_dir.find(":") == -1: + container.volumes.append('${C_LIGHTNING_DATA_DIR}:' + container.c_lightning_mount_dir) + del container.c_lightning_mount_dir + + return app + +def createComposeConfigFromV2(app: dict, nodeRoot: str): + envFile = os.path.join(nodeRoot, ".env") + networkingFile = os.path.join(nodeRoot, "apps", "networking.json") + + newApp: App = generateApp(app) + newApp = convertContainerPermissions(newApp) + validateEnv(newApp) + newApp = convertDataDirToVolume(newApp) + newApp = configureIps(newApp, networkingFile, envFile) + newApp = configureMainPort(newApp, nodeRoot) + configureHiddenServices(newApp, nodeRoot) + finalConfig: AppStage4 = convertContainersToServices(newApp) + newApp = classToDict(finalConfig) + del newApp['metadata'] + if "version" in newApp: + del newApp["version"] + # Set version to 3.8 (current compose file version) + newApp = {'version': '3.8', **newApp} + return newApp diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py new file mode 100644 index 0000000..52ca1be --- /dev/null +++ b/app/lib/composegenerator/v2/networking.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container +from lib.citadelutils import parse_dotenv +import json +from os import path +import random +from lib.composegenerator.v2.utils.networking import getContainerHiddenService +from lib.composegenerator.v1.networking import assignIp, assignPort + + +def getMainContainer(app: App) -> Container: + if len(app.containers) == 1: + return app.containers[0] + else: + if not app.metadata.mainContainer: + app.metadata.mainContainer = 'main' + for container in app.containers: + # Main is recommended, support web for easier porting from Umbrel + if container.name == 'main' or container.name == 'web': + return container + # Fallback to first container + return app.containers[0] + + +def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: + registryFile = path.join(nodeRoot, "apps", "registry.json") + registry: list = [] + if path.isfile(registryFile): + with open(registryFile, 'r') as f: + registry = json.load(f) + else: + raise Exception("Registry file not found") + + dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) + + mainContainer = getMainContainer(app) + + portDetails = assignPort(mainContainer, app.metadata.id, path.join( + nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) + containerPort = portDetails['port'] + portAsEnvVar = portDetails['env_var'] + portToAppend = portAsEnvVar + + mainPort = False + + if mainContainer.port: + portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port) + mainPort = mainContainer.port + del mainContainer.port + else: + portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar) + + if mainContainer.ports: + mainContainer.ports.append(portToAppend) + # Set the main port to the first port in the list, if it contains a :, it's the port after the : + # If it doesn't contain a :, it's the port itself + if mainPort == False: + mainPort = mainContainer.ports[0] + if mainPort.find(":") != -1: + mainPort = mainPort.split(":")[1] + else: + mainContainer.ports = [portToAppend] + if mainPort == False: + mainPort = portDetails['port'] + + mainContainer = assignIp(mainContainer, app.metadata.id, path.join( + nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) + + # If the IP wasn't in dotenv before, now it should be + dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) + + containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace( + "-", "_"), mainContainer.name.upper().replace("-", "_"))] + + # Also set the port in metadata + app.metadata.port = int(containerPort) + if mainPort: + app.metadata.internalPort = int(mainPort) + else: + app.metadata.internalPort = int(containerPort) + + for registryApp in registry: + if registryApp['id'] == app.metadata.id: + registry[registry.index(registryApp)]['port'] = int(containerPort) + registry[registry.index(registryApp)]['internalPort'] = app.metadata.internalPort + break + + with open(registryFile, 'w') as f: + json.dump(registry, f, indent=4, sort_keys=True) + + return app + +def configureIps(app: AppStage2, networkingFile: str, envFile: str): + for container in app.containers: + if container.noNetwork: + # Check if port is defined for the container + if container.port: + raise Exception("Port defined for container without network") + if getMainContainer(app).name == container.name: + raise Exception("Main container without network") + # Skip this iteration of the loop + continue + + container = assignIp(container, app.metadata.id, + networkingFile, envFile) + + return app + + +def configureHiddenServices(app: AppStage3, nodeRoot: str) -> None: + dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) + hiddenServices = "" + + mainContainer = getMainContainer(app) + + for container in app.containers: + env_var = "APP_{}_{}_IP".format( + app.metadata.id.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + hiddenServices += getContainerHiddenService( + app.metadata, container, dotEnv[env_var], container.name == mainContainer.name) + + torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] + torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] + with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: + f.write(hiddenServices) diff --git a/app/lib/composegenerator/v2/types.py b/app/lib/composegenerator/v2/types.py new file mode 100644 index 0000000..2654e1f --- /dev/null +++ b/app/lib/composegenerator/v2/types.py @@ -0,0 +1,153 @@ +from typing import Union +from dataclasses import dataclass, field +from dacite import from_dict + +@dataclass +class Metadata: + id: str + name: str + version: str + category: str + tagline: str + description: str + developer: str + website: str + repo: str + support: str + gallery: list[str] = field(default_factory=list) + dependencies: list[str] = field(default_factory=list) + updateContainer: Union[str, Union[list, None]] = field(default_factory=list) + path: str = "" + defaultPassword: str = "" + torOnly: bool = False + lightningImplementation: Union[str, None] = None + # Added automatically later + port: int = 0 + internalPort: int = 0 + +@dataclass +class Container: + name: str + image: str + permissions: list = field(default_factory=list) + ports: list = field(default_factory=list) + port: Union[int, None] = None + environment: Union[dict, None] = None + data: list = field(default_factory=list) + user: Union[str, None] = None + stop_grace_period: str = '1m' + depends_on: list = field(default_factory=list) + entrypoint: Union[list[str], str] = field(default_factory=list) + bitcoin_mount_dir: Union[str, None] = None + lnd_mount_dir: Union[str, None] = None + c_lightning_mount_dir: Union[str, None] = None + command: Union[list[str], str] = field(default_factory=list) + init: Union[bool, None] = None + stop_signal: Union[str, None] = None + noNetwork: Union[bool, None] = None + hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list) + environment_allow: list = field(default_factory=list) + # Only added later + volumes: list = field(default_factory=list) + restart: Union[str, None] = None + +@dataclass +class App: + version: Union[str, int] + metadata: Metadata + containers: list[Container] + +# Generate an app instance from an app dict +def generateApp(appDict): + return from_dict(data_class=App, data=appDict) + +@dataclass +class Network: + ipv4_address: Union[str, None] = None + +@dataclass +class NetworkConfig: + default: Network + +# After converting data dir and defining volumes, stage 2 +@dataclass +class ContainerStage2: + id: str + name: str + image: str + permissions: list[str] = field(default_factory=list) + ports: list = field(default_factory=list) + environment: Union[dict, None] = None + user: Union[str, None] = None + stop_grace_period: str = '1m' + depends_on: list[str] = field(default_factory=list) + entrypoint: Union[list[str], str] = field(default_factory=list) + command: Union[list[str], str] = field(default_factory=list) + init: Union[bool, None] = None + stop_signal: Union[str, None] = None + noNetwork: Union[bool, None] = None + hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list) + volumes: list[str] = field(default_factory=list) + networks: NetworkConfig = field(default_factory=NetworkConfig) + restart: Union[str, None] = None + +@dataclass +class AppStage2: + version: Union[str, int] + metadata: Metadata + containers: list[ContainerStage2] + +@dataclass +class MetadataStage3: + id: str + name: str + version: str + category: str + tagline: str + description: str + developer: str + website: str + dependencies: list[str] + repo: str + support: str + gallery: list[str] + updateContainer: Union[str, Union[list, None]] = field(default_factory=list) + path: str = "" + defaultPassword: str = "" + torOnly: bool = False + lightningImplementation: Union[str, None] = None + # Added automatically later + port: int = 0 + internalPort: int = 0 + +@dataclass +class AppStage3: + version: Union[str, int] + metadata: MetadataStage3 + containers: list[ContainerStage2] + +@dataclass +class ContainerStage4: + id: str + name: str + image: str + ports: list = field(default_factory=list) + environment: Union[dict, None] = None + user: Union[str, None] = None + stop_grace_period: str = '1m' + depends_on: list[str] = field(default_factory=list) + entrypoint: Union[list[str], str] = field(default_factory=list) + command: Union[list[str], str] = field(default_factory=list) + init: Union[bool, None] = None + stop_signal: Union[str, None] = None + noNetwork: Union[bool, None] = None + hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list) + volumes: list[str] = field(default_factory=list) + networks: NetworkConfig = field(default_factory=NetworkConfig) + restart: Union[str, None] = None + +@dataclass +class AppStage4: + version: Union[str, int] + metadata: MetadataStage3 + services: list[ContainerStage4] \ No newline at end of file diff --git a/app/lib/composegenerator/v2/utils/networking.py b/app/lib/composegenerator/v2/utils/networking.py new file mode 100644 index 0000000..e166597 --- /dev/null +++ b/app/lib/composegenerator/v2/utils/networking.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +from lib.composegenerator.v2.types import Metadata, Container + + +def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str: + hiddenServices = """ +# {} Hidden Service +HiddenServiceDir /var/lib/tor/app-{} +""".format( + name, id + ) + for port in ports: + hiddenServices += "HiddenServicePort {} {}:{}".format(port, internalIp, port) + hiddenServices += "\n" + return hiddenServices + + +def getHiddenServiceString( + name: str, id: str, internalPort, internalIp: str, publicPort +) -> str: + return """ +# {} Hidden Service +HiddenServiceDir /var/lib/tor/app-{} +HiddenServicePort {} {}:{} + +""".format( + name, id, publicPort, internalIp, internalPort + ) + + +def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str: + return getHiddenServiceString(appName, appId, appPort, appIp, "80") + + +def getContainerHiddenService( + metadata: Metadata, container: Container, containerIp: str, isMainContainer: bool +) -> str: + if isMainContainer and not container.hiddenServicePorts: + return getHiddenServiceString( + metadata.name, metadata.id, metadata.internalPort, containerIp, 80 + ) + + if container.hiddenServicePorts: + if isinstance(container.hiddenServicePorts, int): + return getHiddenServiceString( + "{} {}".format(metadata.name, container.name), + metadata.id, + container.hiddenServicePorts, + containerIp, + container.hiddenServicePorts, + ) + elif isinstance(container.hiddenServicePorts, list): + return getHiddenServiceMultiPort( + "{} {}".format(metadata.name, container.name), + metadata.id, + containerIp, + container.hiddenServicePorts, + ) + elif isinstance(container.hiddenServicePorts, dict): + additionalHiddenServices = {} + hiddenServices = "# {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format( + metadata.name, container.name, metadata.id, container.name + ) + for key, value in container.hiddenServicePorts.items(): + if isinstance(key, int): + hiddenServices += "HiddenServicePort {} {}:{}".format( + key, containerIp, value + ) + hiddenServices += "\n" + else: + additionalHiddenServices[key] = value + for key, value in additionalHiddenServices.items(): + hiddenServices += "\n" + if isinstance(value, int): + hiddenServices += "# {} {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format( + metadata.name, container.name, key, metadata.id, container.name + ) + hiddenServices += "HiddenServicePort {} {}:{}".format( + key, containerIp, value + ) + elif isinstance(value, list): + hiddenServices += getHiddenServiceMultiPort( + key, metadata.id, containerIp, value + ) + return hiddenServices + del container.hiddenServicePorts + + return "" diff --git a/app/lib/manage.py b/app/lib/manage.py index 2ca2bc2..04c78dd 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -28,6 +28,7 @@ except Exception: print("Like checking for app updates") from lib.composegenerator.v1.generate import createComposeConfigFromV1 +from lib.composegenerator.v2.generate import createComposeConfigFromV2 from lib.validate import findAndValidateApps from lib.metadata import getAppRegistry from lib.entropy import deriveEntropy @@ -176,6 +177,8 @@ def getApp(appFile: str, appId: str): if 'version' in app and str(app['version']) == "1": return createComposeConfigFromV1(app, nodeRoot) + elif 'version' in app and str(app['version']) == "2": + return createComposeConfigFromV2(app, nodeRoot) else: raise Exception("Error: Unsupported version of app.yml") diff --git a/app/lib/validate.py b/app/lib/validate.py index 43734d8..07a3acf 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -14,6 +14,8 @@ scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") def validateApp(app: dict): with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f: schemaVersion1 = yaml.safe_load(f) + with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: + schemaVersion2 = yaml.safe_load(f) if 'version' in app and str(app['version']) == "1": try: @@ -23,6 +25,14 @@ def validateApp(app: dict): except Exception as e: print(e) return False + elif 'version' in app and str(app['version']) == "2": + try: + validate(app, schemaVersion2) + return True + # Catch and log any errors, and return false + except Exception as e: + print(e) + return False else: print("Unsupported app version") return False