app.yml standard v2

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.formatting.provider": "black"
+}

app/app-standard-v2.yml

@@ -0,0 +1,199 @@
+# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema
+$schema: https://json-schema.org/draft/2020-12/schema
+
+
+title: Citadel app.yml v2
+description: The second revision of Citadel's app.yml format
+type: object
+
+properties:
+  version:
+    type:
+      - string
+      - number
+    description: The version of the app.yml format you're using.
+
+  metadata:
+    type: object
+    properties:
+      name:
+        description: Displayed name of the app
+        type: string
+      version:
+        description: Displayed version for the app
+        type: string
+      category:
+        description: The category you'd put the app in
+        type: string
+      tagline:
+        description: A clever tagline
+        type: string
+      description:
+        description: A longer description of the app
+        type: string
+      developer:
+        description: The awesome people behind the app
+        type: string
+      website:
+        description: Displayed version for the app
+        type: string
+      dependencies:
+        description: The services the app depends on
+        type: array
+        items:
+          type: string
+      repo:
+        description: The development repository for your app
+        type: string
+      support:
+        description: A link to the app support wiki/chat/...
+        type: string
+      gallery:
+        type: array
+        description: >-
+          URLs or paths in the runcitadel/app-images/[app-name] folder with app
+          images
+        items:
+          type: string
+      path:
+        description: The path of the app's visible site the open button should open
+        type: string
+      defaultPassword:
+        description: The app's default password Set this to $APP_SEED if the password is the environment variable $APP_SEED.
+        type: string
+      torOnly:
+        description: Whether the app is only available over tor
+        type: boolean
+      updateContainer:
+        type:
+          - string
+          - array
+        description: The container(s) the developer system should automatically update.
+      lightningImplementation:
+        description: The supported lightning implementation for this app. If your app supports multiple, please publish a separate app.yml for each implementation.
+        type: string
+        enum:
+          - lnd
+          - c-lightning
+    required:
+      - name
+      - version
+      - category
+      - tagline
+      - description
+      - developer
+      - website
+      - repo
+      - support
+      - gallery
+    additionalProperties: false
+
+  containers:
+    type: array
+    items:
+      type: object
+      properties:
+        name:
+          type: string
+        image:
+          type: string
+        permissions:
+          type: array
+          items:
+            type: string
+            enum:
+              - lnd
+              - c-lightning
+              - bitcoind
+              - electrum
+              - root
+              - hw
+        ports:
+          type: array
+          items:
+            type:
+              - string
+              - number
+        port:
+          type: number
+          description: >-
+            If this is the main container, the port inside the container which
+            will be exposed to the outside as the port specified in metadata.
+            If this is not set, the port is passed as an env variable in the format APP_${APP_NAME}_${CONTAINER_NAME}_PORT
+        environment:
+          type: object
+        data:
+          type: array
+          description: >-
+            An array of at directories in the container the app stores its data
+            in. Can be empty. Please only list top-level directories.
+          items:
+            type: string
+        user:
+          type: string
+          description: The user the container should run as
+        stop_grace_period:
+          type: string
+          description: The grace period for stopping the container. Defaults to 1 minute.
+        depends_on:
+          type: array
+          description: The services the container depends on
+        entrypoint:
+          type:
+            - string
+            - array
+          description: The entrypoint for the container
+        bitcoin_mount_dir:
+          type: string
+          description: Where to mount the bitcoin dir
+        lnd_mount_dir:
+          type: string
+          description: Where to mount the lnd dir
+        c_lightning_mount_dir:
+          type: string
+          description: Where to mount the c-lightning dir
+        command:
+          type:
+            - string
+            - array
+          description: The command for the container
+        init:
+          type: boolean
+          description: Whether the container should be run with init
+        stop_signal:
+          type: string
+          description: The signal to send to the container when stopping
+        noNetwork:
+          type: boolean
+          description: >-
+            Set this to true if the container shouldn't get an IP & port
+            exposed. This isn't necessary, but helps the docker-compose.yml generator to generate a cleaner output.
+        hiddenServicePorts:
+          type: 
+            - object
+            - number
+            - array
+          items:
+            type:
+              - string
+              - number
+              - array
+          description: >-
+            This can either be a map of hidden service names (human readable names, not the .onion URL, and strings, not numbers)
+            to a port if your app needs multiple hidden services on different ports,
+            a map of port inside to port on the hidden service (if your app has multiple ports on one hidden service),
+            or simply one port number if your apps hidden service should only expose one port to the outside which isn't 80.
+        restart:
+          type: string
+          description: When the container should restart. Can be 'always' or 'on-failure'.
+      additionalProperties: false
+      required:
+        - name
+        - image
+    additionalProperties: false
+
+required:
+  - metadata
+  - containers
+
+additionalProperties: false

app/lib/composegenerator/v1/generate.py

@@ -10,10 +10,6 @@ from lib.citadelutils import classToDict
 import os
 
 def createComposeConfigFromV1(app: dict, nodeRoot: str):
-    if "version" in app:
-        if str(app['version']) != "1":
-            print("Warning: app version is not supported")
-            return False
     envFile = os.path.join(nodeRoot, ".env")
     networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
 

app/lib/composegenerator/v2/generate.py

@@ -0,0 +1,54 @@
+# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp
+from lib.composegenerator.v2.networking import configureHiddenServices, configureIps, configureMainPort
+from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
+from lib.composegenerator.shared.env import validateEnv
+from lib.citadelutils import classToDict
+import os
+
+def convertDataDirToVolumeGen2(app: App) -> AppStage2:
+    app = convertDataDirToVolume(app)
+    for container in app.containers:
+        if container.lnd_mount_dir != None:
+            if not 'lnd' in container.permissions:
+                print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name))
+                # Skip this container
+                continue
+            # Also skip the container if container.lnd_mount_dir contains a :
+            if container.lnd_mount_dir.find(":") == -1:
+                container.volumes.append('${LND_DATA_DIR}:' + container.lnd_mount_dir)
+            del container.lnd_mount_dir
+        if container.c_lightning_mount_dir != None:
+            if not 'lnd' in container.permissions:
+                print("Warning: container {} of app {} defines c_lightning_mount_dir but doesn't request c-lightning permission".format(container.name, app.metadata.name))
+                # Skip this container
+                continue
+            # Also skip the container if container.c_lightning.mount_dir contains a :
+            if container.c_lightning_mount_dir.find(":") == -1:
+                container.volumes.append('${C_LIGHTNING_DATA_DIR}:' + container.c_lightning_mount_dir)
+            del container.c_lightning_mount_dir
+                
+    return app
+
+def createComposeConfigFromV2(app: dict, nodeRoot: str):
+    envFile = os.path.join(nodeRoot, ".env")
+    networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
+
+    newApp: App = generateApp(app)
+    newApp = convertContainerPermissions(newApp)
+    validateEnv(newApp)
+    newApp = convertDataDirToVolume(newApp)
+    newApp = configureIps(newApp, networkingFile, envFile)
+    newApp = configureMainPort(newApp, nodeRoot)
+    configureHiddenServices(newApp, nodeRoot)
+    finalConfig: AppStage4 = convertContainersToServices(newApp)
+    newApp = classToDict(finalConfig)
+    del newApp['metadata']
+    if "version" in newApp:
+        del newApp["version"]
+    # Set version to 3.8 (current compose file version)
+    newApp = {'version': '3.8', **newApp}
+    return newApp

app/lib/composegenerator/v2/networking.py

@@ -0,0 +1,130 @@
+# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container
+from lib.citadelutils import parse_dotenv
+import json
+from os import path
+import random
+from lib.composegenerator.v2.utils.networking import getContainerHiddenService
+from lib.composegenerator.v1.networking import assignIp, assignPort
+
+
+def getMainContainer(app: App) -> Container:
+    if len(app.containers) == 1:
+        return app.containers[0]
+    else:
+        if not app.metadata.mainContainer:
+            app.metadata.mainContainer = 'main'
+        for container in app.containers:
+            # Main is recommended, support web for easier porting from Umbrel
+            if container.name == 'main' or container.name == 'web':
+                return container
+    # Fallback to first container
+    return app.containers[0]
+
+
+def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
+    registryFile = path.join(nodeRoot, "apps", "registry.json")
+    registry: list = []
+    if path.isfile(registryFile):
+        with open(registryFile, 'r') as f:
+            registry = json.load(f)
+    else:
+        raise Exception("Registry file not found")
+
+    dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
+
+    mainContainer = getMainContainer(app)
+
+    portDetails = assignPort(mainContainer, app.metadata.id, path.join(
+        nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
+    containerPort = portDetails['port']
+    portAsEnvVar = portDetails['env_var']
+    portToAppend = portAsEnvVar
+
+    mainPort = False
+
+    if mainContainer.port:
+        portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port)
+        mainPort = mainContainer.port
+        del mainContainer.port
+    else:
+        portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar)
+
+    if mainContainer.ports:
+        mainContainer.ports.append(portToAppend)
+        # Set the main port to the first port in the list, if it contains a :, it's the port after the :
+        # If it doesn't contain a :, it's the port itself
+        if mainPort == False:
+            mainPort = mainContainer.ports[0]
+            if mainPort.find(":") != -1:
+                mainPort = mainPort.split(":")[1]
+    else:
+        mainContainer.ports = [portToAppend]
+        if mainPort == False:
+            mainPort = portDetails['port']
+
+    mainContainer = assignIp(mainContainer, app.metadata.id, path.join(
+        nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
+
+    # If the IP wasn't in dotenv before, now it should be
+    dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
+
+    containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace(
+        "-", "_"), mainContainer.name.upper().replace("-", "_"))]
+
+    # Also set the port in metadata
+    app.metadata.port = int(containerPort)
+    if mainPort:
+        app.metadata.internalPort = int(mainPort)
+    else:
+        app.metadata.internalPort = int(containerPort)
+
+    for registryApp in registry:
+        if registryApp['id'] == app.metadata.id:
+            registry[registry.index(registryApp)]['port'] = int(containerPort)
+            registry[registry.index(registryApp)]['internalPort'] = app.metadata.internalPort
+            break
+
+    with open(registryFile, 'w') as f:
+        json.dump(registry, f, indent=4, sort_keys=True)
+
+    return app
+
+def configureIps(app: AppStage2, networkingFile: str, envFile: str):
+    for container in app.containers:
+        if container.noNetwork:
+            # Check if port is defined for the container
+            if container.port:
+                raise Exception("Port defined for container without network")
+            if getMainContainer(app).name == container.name:
+                raise Exception("Main container without network")
+            # Skip this iteration of the loop
+            continue
+
+        container = assignIp(container, app.metadata.id,
+                             networkingFile, envFile)
+
+    return app
+
+
+def configureHiddenServices(app: AppStage3, nodeRoot: str) -> None:
+    dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
+    hiddenServices = ""
+
+    mainContainer = getMainContainer(app)
+
+    for container in app.containers:
+        env_var = "APP_{}_{}_IP".format(
+            app.metadata.id.upper().replace("-", "_"),
+            container.name.upper().replace("-", "_")
+        )
+        hiddenServices += getContainerHiddenService(
+            app.metadata, container, dotEnv[env_var], container.name == mainContainer.name)
+
+    torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
+    torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
+    with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
+        f.write(hiddenServices)

app/lib/composegenerator/v2/types.py

@@ -0,0 +1,153 @@
+from typing import Union
+from dataclasses import dataclass, field
+from dacite import from_dict
+
+@dataclass
+class Metadata:
+    id: str
+    name: str
+    version: str
+    category: str
+    tagline: str
+    description: str
+    developer: str
+    website: str
+    repo: str
+    support: str
+    gallery: list[str] = field(default_factory=list)
+    dependencies: list[str] = field(default_factory=list)
+    updateContainer: Union[str, Union[list, None]] = field(default_factory=list)
+    path: str = ""
+    defaultPassword: str = ""
+    torOnly: bool = False
+    lightningImplementation: Union[str, None] = None
+    # Added automatically later
+    port: int = 0
+    internalPort: int = 0
+
+@dataclass
+class Container:
+    name: str
+    image: str
+    permissions: list = field(default_factory=list)
+    ports: list = field(default_factory=list)
+    port: Union[int, None] = None
+    environment: Union[dict, None] = None
+    data: list = field(default_factory=list)
+    user: Union[str, None] = None
+    stop_grace_period: str = '1m'
+    depends_on: list = field(default_factory=list)
+    entrypoint: Union[list[str], str] = field(default_factory=list)
+    bitcoin_mount_dir: Union[str, None] = None
+    lnd_mount_dir: Union[str, None] = None
+    c_lightning_mount_dir: Union[str, None] = None
+    command: Union[list[str], str] = field(default_factory=list)
+    init: Union[bool, None] = None
+    stop_signal: Union[str, None] = None
+    noNetwork: Union[bool, None] = None
+    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
+    environment_allow: list = field(default_factory=list)
+    # Only added later
+    volumes: list = field(default_factory=list)
+    restart: Union[str, None] = None
+
+@dataclass
+class App:
+    version: Union[str, int]
+    metadata: Metadata
+    containers: list[Container]
+
+# Generate an app instance from an app dict
+def generateApp(appDict):
+    return from_dict(data_class=App, data=appDict)
+
+@dataclass
+class Network:
+    ipv4_address: Union[str, None] = None
+
+@dataclass
+class NetworkConfig:
+    default: Network
+
+# After converting data dir and defining volumes, stage 2
+@dataclass
+class ContainerStage2:
+    id: str
+    name: str
+    image: str
+    permissions: list[str] = field(default_factory=list)
+    ports: list = field(default_factory=list)
+    environment: Union[dict, None] = None
+    user: Union[str, None] = None
+    stop_grace_period: str = '1m'
+    depends_on: list[str] = field(default_factory=list)
+    entrypoint: Union[list[str], str] = field(default_factory=list)
+    command: Union[list[str], str] = field(default_factory=list)
+    init: Union[bool, None] = None
+    stop_signal: Union[str, None] = None
+    noNetwork: Union[bool, None] = None
+    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
+    volumes: list[str] = field(default_factory=list)
+    networks: NetworkConfig = field(default_factory=NetworkConfig)
+    restart: Union[str, None] = None
+
+@dataclass
+class AppStage2:
+    version: Union[str, int]
+    metadata: Metadata
+    containers: list[ContainerStage2]
+
+@dataclass
+class MetadataStage3:
+    id: str
+    name: str
+    version: str
+    category: str
+    tagline: str
+    description: str
+    developer: str
+    website: str
+    dependencies: list[str]
+    repo: str
+    support: str
+    gallery: list[str]
+    updateContainer: Union[str, Union[list, None]] = field(default_factory=list)
+    path: str = ""
+    defaultPassword: str = ""
+    torOnly: bool = False
+    lightningImplementation: Union[str, None] = None
+    # Added automatically later
+    port: int = 0
+    internalPort: int = 0
+
+@dataclass
+class AppStage3:
+    version: Union[str, int]
+    metadata: MetadataStage3
+    containers: list[ContainerStage2]
+
+@dataclass
+class ContainerStage4:
+    id: str
+    name: str
+    image: str
+    ports: list = field(default_factory=list)
+    environment: Union[dict, None] = None
+    user: Union[str, None] = None
+    stop_grace_period: str = '1m'
+    depends_on: list[str] = field(default_factory=list)
+    entrypoint: Union[list[str], str] = field(default_factory=list)
+    command: Union[list[str], str] = field(default_factory=list)
+    init: Union[bool, None] = None
+    stop_signal: Union[str, None] = None
+    noNetwork: Union[bool, None] = None
+    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
+    volumes: list[str] = field(default_factory=list)
+    networks: NetworkConfig = field(default_factory=NetworkConfig)
+    restart: Union[str, None] = None
+
+@dataclass
+class AppStage4:
+    version: Union[str, int]
+    metadata: MetadataStage3
+    services: list[ContainerStage4]

app/lib/composegenerator/v2/utils/networking.py

@@ -0,0 +1,91 @@
+# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from lib.composegenerator.v2.types import Metadata, Container
+
+
+def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str:
+    hiddenServices = """
+# {} Hidden Service
+HiddenServiceDir /var/lib/tor/app-{}
+""".format(
+        name, id
+    )
+    for port in ports:
+        hiddenServices += "HiddenServicePort {} {}:{}".format(port, internalIp, port)
+        hiddenServices += "\n"
+    return hiddenServices
+
+
+def getHiddenServiceString(
+    name: str, id: str, internalPort, internalIp: str, publicPort
+) -> str:
+    return """
+# {} Hidden Service
+HiddenServiceDir /var/lib/tor/app-{}
+HiddenServicePort {} {}:{}
+
+""".format(
+        name, id, publicPort, internalIp, internalPort
+    )
+
+
+def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str:
+    return getHiddenServiceString(appName, appId, appPort, appIp, "80")
+
+
+def getContainerHiddenService(
+    metadata: Metadata, container: Container, containerIp: str, isMainContainer: bool
+) -> str:
+    if isMainContainer and not container.hiddenServicePorts:
+        return getHiddenServiceString(
+            metadata.name, metadata.id, metadata.internalPort, containerIp, 80
+        )
+
+    if container.hiddenServicePorts:
+        if isinstance(container.hiddenServicePorts, int):
+            return getHiddenServiceString(
+                "{} {}".format(metadata.name, container.name),
+                metadata.id,
+                container.hiddenServicePorts,
+                containerIp,
+                container.hiddenServicePorts,
+            )
+        elif isinstance(container.hiddenServicePorts, list):
+            return getHiddenServiceMultiPort(
+                "{} {}".format(metadata.name, container.name),
+                metadata.id,
+                containerIp,
+                container.hiddenServicePorts,
+            )
+        elif isinstance(container.hiddenServicePorts, dict):
+            additionalHiddenServices = {}
+            hiddenServices = "# {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format(
+                metadata.name, container.name, metadata.id, container.name
+            )
+            for key, value in container.hiddenServicePorts.items():
+                if isinstance(key, int):
+                    hiddenServices += "HiddenServicePort {} {}:{}".format(
+                        key, containerIp, value
+                    )
+                    hiddenServices += "\n"
+                else:
+                    additionalHiddenServices[key] = value
+            for key, value in additionalHiddenServices.items():
+                hiddenServices += "\n"
+                if isinstance(value, int):
+                    hiddenServices += "# {} {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format(
+                        metadata.name, container.name, key, metadata.id, container.name
+                    )
+                    hiddenServices += "HiddenServicePort {} {}:{}".format(
+                        key, containerIp, value
+                    )
+                elif isinstance(value, list):
+                    hiddenServices += getHiddenServiceMultiPort(
+                        key, metadata.id, containerIp, value
+                    )
+            return hiddenServices
+        del container.hiddenServicePorts
+
+    return ""

app/lib/manage.py

@@ -28,6 +28,7 @@ except Exception:
     print("Like checking for app updates")
 
 from lib.composegenerator.v1.generate import createComposeConfigFromV1
+from lib.composegenerator.v2.generate import createComposeConfigFromV2
 from lib.validate import findAndValidateApps
 from lib.metadata import getAppRegistry
 from lib.entropy import deriveEntropy
@@ -176,6 +177,8 @@ def getApp(appFile: str, appId: str):
 
     if 'version' in app and str(app['version']) == "1":
         return createComposeConfigFromV1(app, nodeRoot)
+    elif 'version' in app and str(app['version']) == "2":
+        return createComposeConfigFromV2(app, nodeRoot)
     else:
         raise Exception("Error: Unsupported version of app.yml")
 

app/lib/validate.py

@@ -14,6 +14,8 @@ scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
 def validateApp(app: dict):
     with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f:
         schemaVersion1 = yaml.safe_load(f)
+    with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f:
+        schemaVersion2 = yaml.safe_load(f)
 
     if 'version' in app and str(app['version']) == "1":
         try:
@@ -23,6 +25,14 @@ def validateApp(app: dict):
         except Exception as e:
             print(e)
             return False
+    elif 'version' in app and str(app['version']) == "2":
+        try:
+            validate(app, schemaVersion2)
+            return True
+        # Catch and log any errors, and return false
+        except Exception as e:
+            print(e)
+            return False
     else:
         print("Unsupported app version")
         return False