Remove legacy compose generator entirely

app/app-standard-v2.yml

@@ -1,201 +0,0 @@
-# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema
-$schema: https://json-schema.org/draft/2020-12/schema
-
-
-title: Citadel app.yml v2
-description: The second revision of Citadel's app.yml format
-type: object
-
-properties:
-  version:
-    type:
-      - string
-      - number
-    description: The version of the app.yml format you're using.
-
-  metadata:
-    type: object
-    properties:
-      name:
-        description: Displayed name of the app
-        type: string
-      version:
-        description: Displayed version for the app
-        type: string
-      category:
-        description: The category you'd put the app in
-        type: string
-      tagline:
-        description: A clever tagline
-        type: string
-      description:
-        description: A longer description of the app
-        type: string
-      developer:
-        description: The awesome people behind the app
-        type: string
-      website:
-        description: Displayed version for the app
-        type: string
-      dependencies:
-        description: The services the app depends on
-        type: array
-        items:
-          type: string
-      repo:
-        description: The development repository for your app
-        type: string
-      support:
-        description: A link to the app support wiki/chat/...
-        type: string
-      gallery:
-        type: array
-        description: >-
-          URLs or paths in the runcitadel/app-images/[app-name] folder with app
-          images
-        items:
-          type: string
-      path:
-        description: The path of the app's visible site the open button should open
-        type: string
-      defaultPassword:
-        description: The app's default password Set this to $APP_SEED if the password is the environment variable $APP_SEED.
-        type: string
-      torOnly:
-        description: Whether the app is only available over tor
-        type: boolean
-      updateContainer:
-        type:
-          - string
-          - array
-        description: The container(s) the developer system should automatically update.
-      lightningImplementation:
-        description: The supported lightning implementation for this app. If your app supports multiple, please publish a separate app.yml for each implementation.
-        type: string
-        enum:
-          - lnd
-          - c-lightning
-    required:
-      - name
-      - version
-      - category
-      - tagline
-      - description
-      - developer
-      - website
-      - repo
-      - support
-      - gallery
-    additionalProperties: false
-
-  containers:
-    type: array
-    items:
-      type: object
-      properties:
-        name:
-          type: string
-        image:
-          type: string
-        permissions:
-          type: array
-          items:
-            type: string
-            enum:
-              - lnd
-              - c-lightning
-              - bitcoind
-              - electrum
-              - root
-              - hw
-        ports:
-          type: array
-          items:
-            type:
-              - string
-              - number
-        port:
-          type: number
-          description: >-
-            If this is the main container, the port inside the container which
-            will be exposed to the outside as the port specified in metadata.
-            If this is not set, the port is passed as an env variable in the format APP_${APP_NAME}_${CONTAINER_NAME}_PORT
-        environment:
-          type: object
-        data:
-          type: array
-          description: >-
-            An array of at directories in the container the app stores its data
-            in. Can be empty. Please only list top-level directories.
-          items:
-            type: string
-        user:
-          type: string
-          description: The user the container should run as
-        stop_grace_period:
-          type: string
-          description: The grace period for stopping the container. Defaults to 1 minute.
-        depends_on:
-          type: array
-          description: The services the container depends on
-        entrypoint:
-          type:
-            - string
-            - array
-          description: The entrypoint for the container
-        bitcoin_mount_dir:
-          type: string
-          description: Where to mount the bitcoin dir
-        lnd_mount_dir:
-          type: string
-          description: Where to mount the lnd dir
-        c_lightning_mount_dir:
-          type: string
-          description: Where to mount the c-lightning dir
-        command:
-          type:
-            - string
-            - array
-          description: The command for the container
-        init:
-          type: boolean
-          description: Whether the container should be run with init
-        stop_signal:
-          type: string
-          description: The signal to send to the container when stopping
-        noNetwork:
-          type: boolean
-          description: >-
-            Set this to true if the container shouldn't get an IP & port
-            exposed. This isn't necessary, but helps the docker-compose.yml generator to generate a cleaner output.
-        hiddenServicePorts:
-          type: 
-            - object
-            - number
-            - array
-          items:
-            type:
-              - string
-              - number
-              - array
-          description: >-
-            This can either be a map of hidden service names (human readable names, not the .onion URL, and strings, not numbers)
-            to a port if your app needs multiple hidden services on different ports,
-            a map of port inside to port on the hidden service (if your app has multiple ports on one hidden service),
-            or simply one port number if your apps hidden service should only expose one port to the outside which isn't 80.
-        restart:
-          type: string
-          description: When the container should restart. Can be 'always' or 'on-failure'.
-        network_mode:
-          type: string
-      additionalProperties: false
-      required:
-        - name
-        - image
-    additionalProperties: false
-
-required:
-  - metadata
-  - containers
-
-additionalProperties: false

app/lib/composegenerator/shared/const.py

@@ -1,53 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-def permissions():
-    return {
-        "lnd": {
-            "environment_allow": [
-                "LND_IP",
-                "LND_GRPC_PORT",
-                "LND_REST_PORT",
-                "BITCOIN_NETWORK"
-            ],
-            "volumes": [
-                '${LND_DATA_DIR}:/lnd:ro'
-            ]
-        },
-        "bitcoind": {
-            "environment_allow": [
-                "BITCOIN_IP",
-                "BITCOIN_NETWORK",
-                "BITCOIN_P2P_PORT",
-                "BITCOIN_RPC_PORT",
-                "BITCOIN_RPC_USER",
-                "BITCOIN_RPC_PASS",
-                "BITCOIN_RPC_AUTH",
-                "BITCOIN_ZMQ_RAWBLOCK_PORT",
-                "BITCOIN_ZMQ_RAWTX_PORT",
-                "BITCOIN_ZMQ_HASHBLOCK_PORT",
-                "BITCOIN_ZMQ_SEQUENCE_PORT",
-            ],
-            "volumes": [
-                "${BITCOIN_DATA_DIR}:/bitcoin"
-            ]
-        },
-        "electrum": {
-            "environment_allow": [
-                "ELECTRUM_IP",
-                "ELECTRUM_PORT",
-            ],
-            "volumes": []
-        },
-        "c-lightning": {
-            "environment_allow": [
-                "C_LIGHTNING_IP"
-            ],
-            "volumes": []
-        },
-    }
-
-# Vars which are always allowed without permissions
-always_allowed_env = ["TOR_PROXY_IP", "TOR_PROXY_PORT",
-                      "APP_DOMAIN", "APP_HIDDEN_SERVICE", "BITCOIN_NETWORK"]

app/lib/composegenerator/shared/env.py

@@ -1,53 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import re
-from typing import Union
-from lib.composegenerator.v2.types import App
-from lib.composegenerator.shared.const import always_allowed_env
-from lib.citadelutils import checkArrayContainsAllElements, getEnvVars
-
-def validateEnvByValue(env: list, allowed: list, app_name: str):
-    # Combine always_allowed_env with allowed into one list
-    # Then check if all elements in env are in the resulting list
-    all_allowed = allowed + always_allowed_env
-    if not checkArrayContainsAllElements(env, all_allowed):
-        # This has a weird syntax, and it confuses VSCode, but it works
-        validation_regex = r"APP_{}(\S+)".format(
-            app_name.upper().replace("-", "_"))
-        for key in env:
-            # If the key is neither in all_allowed nor is a full match against the validation regex, print a warning and return false
-            if key not in all_allowed and re.fullmatch(validation_regex, key) is None and not key.startswith("APP_HIDDEN_SERVICE")and not key.startswith("APP_SEED"):
-                print("Invalid environment variable {} in app {}".format(
-                    key, app_name))
-                return False
-    return True
-
-def validateEnvStringOrListorDict(env: Union[str, Union[list, dict]], existingEnv: list, app_name: str, container_name: str):
-    envList = []
-    if isinstance(env, dict):
-        envList = env.values()
-    elif isinstance(env, list):
-        envList = env
-    elif isinstance(env, str):
-        envList = [env]
-    for envVar in envList:
-        if not validateEnvByValue(getEnvVars(envVar), existingEnv, app_name):
-            raise Exception("Env var {} not defined for container {} of app {}".format(envVar, container_name, app_name))
-    
-
-def validateEnv(app: App):
-    # For every container of the app, check if all env vars in the strings in environment are defined in env
-    for container in app.containers:
-        if container is not None:
-            if container.environment_allow:
-                existingEnv = container.environment_allow
-                del container.environment_allow
-            else:
-                existingEnv = []
-            if container.environment:
-                validateEnvStringOrListorDict(container.command, existingEnv, app.metadata.id, container.name)
-                validateEnvStringOrListorDict(container.entrypoint, existingEnv, app.metadata.id, container.name)
-                validateEnvStringOrListorDict(container.environment, existingEnv, app.metadata.id, container.name)
-    return app

app/lib/composegenerator/shared/main.py

@@ -1,29 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# Main functions
-from lib.composegenerator.v2.types import App, AppStage3
-from lib.composegenerator.shared.const import permissions
-
-
-def convertContainerPermissions(app: App) -> App:
-    for container in app.containers:
-        for permission in container.permissions:
-            if permission in permissions():
-                container.environment_allow.extend(permissions()[permission]['environment_allow'])
-                container.volumes.extend(permissions()[permission]['volumes'])
-            else:
-                print("Warning: container {} of app {} defines unknown permission {}".format(container.name, app.metadata.name, permission))
-    return app
-
-def convertContainersToServices(app: AppStage3) -> AppStage3:
-    services = {}
-    for container in app.containers:
-        if container.permissions:
-            del container.permissions
-        services[container.name] = container
-        del services[container.name].name
-    del app.containers
-    app.services = services
-    return app

app/lib/composegenerator/shared/networking.py

@@ -1,174 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import json
-from os import path
-import random
-from lib.composegenerator.shared.utils.networking import getContainerHiddenService
-from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container
-from lib.citadelutils import parse_dotenv
-from dacite import from_dict
-
-def getMainContainer(app: App) -> Container:
-    if len(app.containers) == 1:
-        return app.containers[0]
-    else:
-        for container in app.containers:
-            # Main is recommended, support web for easier porting from Umbrel
-            if container.name == 'main' or container.name == 'web':
-                return container
-    # Fallback to first container
-    return app.containers[0]
-
-def assignIpV4(appId: str, containerName: str):
-    scriptDir = path.dirname(path.realpath(__file__))
-    nodeRoot = path.join(scriptDir, "..", "..", "..", "..")
-    networkingFile = path.join(nodeRoot, "apps", "networking.json")
-    envFile = path.join(nodeRoot, ".env")
-    cleanContainerName = containerName.strip()
-    # If the name still contains a newline, throw an error
-    if cleanContainerName.find("\n") != -1:
-        raise Exception("Newline in container name")
-    env_var = "APP_{}_{}_IP".format(
-        appId.upper().replace("-", "_"),
-        cleanContainerName.upper().replace("-", "_")
-    )
-    # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
-    # can be used
-    usedIps = []
-    networkingData = {}
-    if path.isfile(networkingFile):
-        with open(networkingFile, 'r') as f:
-            networkingData = json.load(f)
-
-    if 'ip_addresses' in networkingData:
-        usedIps = list(networkingData['ip_addresses'].values())
-    else:
-        networkingData['ip_addresses'] = {}
-    # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
-    # If the IP is already in use, it will be tried again until it's not in use
-    # If it's not in use, it will be added to the usedIps list and written to the usedIpFile
-    # If the usedIpsFile contains all IPs between  10.21.21.20 and  10.21.21.255 (inclusive),
-    # Throw an error, because no more IPs can be used
-    if len(usedIps) == 235:
-        raise Exception("No more IPs can be used")
-
-    if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']:
-        ip = networkingData['ip_addresses']["{}-{}".format(
-            appId, cleanContainerName)]
-    else:
-        while True:
-            ip = "10.21.21." + str(random.randint(20, 255))
-            if ip not in usedIps:
-                networkingData['ip_addresses']["{}-{}".format(
-                    appId, cleanContainerName)] = ip
-                break
-
-    dotEnv = parse_dotenv(envFile)
-    if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
-        return
-
-    with open(envFile, 'a') as f:
-        f.write("{}={}\n".format(env_var, ip))
-    with open(networkingFile, 'w') as f:
-        json.dump(networkingData, f)
-
-def assignIp(container: ContainerStage2, appId: str) -> ContainerStage2:
-    scriptDir = path.dirname(path.realpath(__file__))
-    nodeRoot = path.join(scriptDir, "..", "..", "..", "..")
-    networkingFile = path.join(nodeRoot, "apps", "networking.json")
-    envFile = path.join(nodeRoot, ".env")
-    # Strip leading/trailing whitespace from container.name
-    container.name = container.name.strip()
-    # If the name still contains a newline, throw an error
-    if container.name.find("\n") != -1:
-        raise Exception("Newline in container name")
-    env_var = "APP_{}_{}_IP".format(
-        appId.upper().replace("-", "_"),
-        container.name.upper().replace("-", "_")
-    )
-    # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
-    # can be used
-    usedIps = []
-    networkingData = {}
-    if path.isfile(networkingFile):
-        with open(networkingFile, 'r') as f:
-            networkingData = json.load(f)
-
-    if 'ip_addresses' in networkingData:
-        usedIps = list(networkingData['ip_addresses'].values())
-    else:
-        networkingData['ip_addresses'] = {}
-    # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
-    # If the IP is already in use, it will be tried again until it's not in use
-    # If it's not in use, it will be added to the usedIps list and written to the usedIpFile
-    # If the usedIpsFile contains all IPs between  10.21.21.20 and  10.21.21.255 (inclusive),
-    # Throw an error, because no more IPs can be used
-    if len(usedIps) == 235:
-        raise Exception("No more IPs can be used")
-
-    if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
-        ip = networkingData['ip_addresses']["{}-{}".format(
-            appId, container.name)]
-    else:
-        while True:
-            ip = "10.21.21." + str(random.randint(20, 255))
-            if ip not in usedIps:
-                networkingData['ip_addresses']["{}-{}".format(
-                    appId, container.name)] = ip
-                break
-    container.networks = from_dict(data_class=NetworkConfig, data={'default': {
-        'ipv4_address': "$" + env_var}})
-
-    dotEnv = parse_dotenv(envFile)
-    if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
-        return container
-
-    # Now append a new line  with APP_{app_name}_{container_name}_IP=${IP} to the envFile
-    with open(envFile, 'a') as f:
-        f.write("{}={}\n".format(env_var, ip))
-    with open(networkingFile, 'w') as f:
-        json.dump(networkingData, f)
-    return container
-
-def configureIps(app: AppStage2, networkingFile: str, envFile: str):
-    for container in app.containers:
-        if container.network_mode and container.network_mode == "host":
-            continue
-        if container.noNetwork:
-            # Check if port is defined for the container
-            if container.port:
-                raise Exception("Port defined for container without network")
-            if getMainContainer(app).name == container.name:
-                raise Exception("Main container without network")
-            # Skip this iteration of the loop
-            continue
-
-        container = assignIp(container, app.metadata.id)
-
-    return app
-
-def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3:
-    dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
-    hiddenServices = ""
-
-    mainContainer = getMainContainer(app)
-
-    for container in app.containers:
-        if container.network_mode and container.network_mode == "host":
-            continue
-        env_var = "APP_{}_{}_IP".format(
-            app.metadata.id.upper().replace("-", "_"),
-            container.name.upper().replace("-", "_")
-        )
-        hiddenServices += getContainerHiddenService(
-            app.metadata, container, dotEnv[env_var], container.name == mainContainer.name)
-        if container.hiddenServicePorts:
-            del container.hiddenServicePorts
-
-    torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
-    torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
-    with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
-        f.write(hiddenServices)
-    return app

app/lib/composegenerator/shared/utils/networking.py

@@ -1,95 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-from lib.composegenerator.v2.types import Metadata, Container
-
-
-def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str:
-    hiddenServices = """
-# {} Hidden Service
-HiddenServiceDir /var/lib/tor/app-{}
-""".format(
-        name, id
-    )
-    for port in ports:
-        hiddenServices += "HiddenServicePort {} {}:{}".format(port, internalIp, port)
-        hiddenServices += "\n"
-    return hiddenServices
-
-
-def getHiddenServiceString(
-    name: str, id: str, internalPort, internalIp: str, publicPort
-) -> str:
-    return """
-# {} Hidden Service
-HiddenServiceDir /var/lib/tor/app-{}
-HiddenServicePort {} {}:{}
-
-""".format(
-        name, id, publicPort, internalIp, internalPort
-    )
-
-
-def getContainerHiddenService(
-    metadata: Metadata, container: Container, containerIp: str, isMainContainer: bool
-) -> str:
-    if isMainContainer and not container.hiddenServicePorts:
-        return getHiddenServiceString(
-            metadata.name, metadata.id, metadata.internalPort, containerIp, 80
-        )
-
-    if container.hiddenServicePorts:
-        if isinstance(container.hiddenServicePorts, int):
-            return getHiddenServiceString(
-                "{} {}".format(metadata.name, container.name),
-                metadata.id if isMainContainer else "{}-{}".format(metadata.id, container.name),
-                container.hiddenServicePorts,
-                containerIp,
-                container.hiddenServicePorts,
-            )
-        elif isinstance(container.hiddenServicePorts, list):
-            return getHiddenServiceMultiPort(
-                "{} {}".format(metadata.name, container.name),
-                metadata.id if isMainContainer else "{}-{}".format(metadata.id, container.name),
-                containerIp,
-                container.hiddenServicePorts,
-            )
-        elif isinstance(container.hiddenServicePorts, dict):
-            additionalHiddenServices = {}
-            hiddenServices = "# {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format(
-                metadata.name, container.name, metadata.id, container.name
-            )
-            initialHiddenServices = "# {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format(
-                metadata.name, container.name, metadata.id, container.name
-            )
-            otherHiddenServices = ""
-            for key, value in container.hiddenServicePorts.items():
-                if isinstance(key, int):
-                    hiddenServices += "HiddenServicePort {} {}:{}".format(
-                        key, containerIp, value
-                    )
-                    hiddenServices += "\n"
-                else:
-                    additionalHiddenServices[key] = value
-            for key, value in additionalHiddenServices.items():
-                if isinstance(value, int):
-                    otherHiddenServices += "# {} {} {} Hidden Service\nHiddenServiceDir /var/lib/tor/app-{}-{}\n".format(
-                        metadata.name, container.name, key, metadata.id, key
-                    )
-                    otherHiddenServices += "HiddenServicePort {} {}:{}".format(
-                        value, containerIp, value
-                    )
-                    otherHiddenServices += "\n"
-                elif isinstance(value, list):
-                    otherHiddenServices += getHiddenServiceMultiPort(
-                        "{} {}".format(metadata.name, key), "{}-{}".format(metadata.id, key), containerIp, value
-                    )
-                
-            if hiddenServices == initialHiddenServices:
-                return otherHiddenServices
-            else :
-                return hiddenServices + "\n" + otherHiddenServices
-        del container.hiddenServicePorts
-
-    return ""

app/lib/composegenerator/v2/generate.py

@@ -1,71 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp
-from lib.composegenerator.v2.networking import configureMainPort
-from lib.composegenerator.shared.networking import configureHiddenServices, configureIps
-from lib.composegenerator.shared.main import convertContainerPermissions, convertContainersToServices
-from lib.composegenerator.shared.env import validateEnv
-from lib.citadelutils import classToDict
-import os
-
-def convertDataDirToVolume(app: App) -> AppStage2:
-    for container in app.containers:
-        # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
-        # Also, a datadir shouldn't start with a /
-        for dataDir in container.data:
-            if dataDir.find("..") == -1 and dataDir[0] != "/":
-                container.volumes.append(
-                    '${APP_DATA_DIR}/' + dataDir)
-            else:
-                print("Data dir " + dataDir +
-                        " contains invalid characters")
-        del container.data
-        if container.bitcoin_mount_dir != None:
-            if not 'bitcoind' in container.permissions:
-                print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
-                # Skip this container
-                continue
-            # Also skip the container if container.bitcoin_mount_dir contains a :
-            if container.bitcoin_mount_dir.find(":") == -1:
-                container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir)
-            del container.bitcoin_mount_dir
-        if container.lnd_mount_dir != None:
-            if not 'lnd' in container.permissions:
-                print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name))
-                # Skip this container
-                continue
-            # Also skip the container if container.lnd_mount_dir contains a :
-            if container.lnd_mount_dir.find(":") == -1:
-                container.volumes.append('${LND_DATA_DIR}:' + container.lnd_mount_dir)
-            del container.lnd_mount_dir
-        if container.c_lightning_mount_dir != None:
-            if not 'lnd' in container.permissions:
-                print("Warning: container {} of app {} defines c_lightning_mount_dir but doesn't request c-lightning permission".format(container.name, app.metadata.name))
-                # Skip this container
-                continue
-            # Also skip the container if container.c_lightning.mount_dir contains a :
-            if container.c_lightning_mount_dir.find(":") == -1:
-                container.volumes.append('${C_LIGHTNING_DATA_DIR}:' + container.c_lightning_mount_dir)
-            del container.c_lightning_mount_dir
-                
-    return app
-
-def createComposeConfigFromV2(app: dict, nodeRoot: str):
-    envFile = os.path.join(nodeRoot, ".env")
-    networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
-
-    newApp: App = generateApp(app)
-    newApp = convertContainerPermissions(newApp)
-    newApp = validateEnv(newApp)
-    newApp = convertDataDirToVolume(newApp)
-    newApp = configureIps(newApp, networkingFile, envFile)
-    newApp = configureMainPort(newApp, nodeRoot)
-    newApp = configureHiddenServices(newApp, nodeRoot)
-    finalConfig: AppStage4 = convertContainersToServices(newApp)
-    newApp = classToDict(finalConfig)
-    del newApp['metadata']
-    if "version" in newApp:
-        del newApp["version"]
-    return newApp

app/lib/composegenerator/v2/networking.py

@@ -1,146 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-from lib.citadelutils import parse_dotenv
-from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container
-import json
-from os import path
-import os
-import random
-from lib.composegenerator.shared.networking import assignIp, getMainContainer
-from lib.citadelutils import FileLock
-
-def getFreePort(networkingFile: str, appId: str):
-    # Ports used currently in Citadel
-    usedPorts = [
-        # Dashboard
-        80,
-        # Sometimes used by nginx with some setups
-        433,
-        # Dashboard SSL
-        443,
-        # Bitcoin Core P2P
-        8333,
-        # LND gRPC
-        10009,
-        # LND REST
-        8080,
-        # Electrum Server
-        50001,
-        # Tor Proxy
-        9050,
-    ]
-    networkingData = {}
-    if path.isfile(networkingFile):
-        with open(networkingFile, 'r') as f:
-            networkingData = json.load(f)
-    if 'ports' in networkingData:
-        usedPorts += list(networkingData['ports'].values())
-    else:
-        networkingData['ports'] = {}
-
-    if appId in networkingData['ports']:
-        return networkingData['ports'][appId]
-
-    while True:
-        port = str(random.randint(1024, 49151))
-        if port not in usedPorts:
-            # Check if anyhing is listening on the specific port
-            if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
-                networkingData['ports'][appId] = port
-                break
-
-    with open(networkingFile, 'w') as f:
-        json.dump(networkingData, f)
-
-    return port
-
-def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
-    # Strip leading/trailing whitespace from container.name
-    container.name = container.name.strip()
-    # If the name still contains a newline, throw an error
-    if container.name.find("\n") != -1 or container.name.find(" ") != -1:
-        raise Exception("Newline or space in container name")
-
-    env_var = "APP_{}_{}_PORT".format(
-        appId.upper().replace("-", "_"),
-        container.name.upper().replace("-", "_")
-    )
-
-    port = getFreePort(networkingFile, appId)
-
-    dotEnv = parse_dotenv(envFile)
-    if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
-        return {"port": port, "env_var": "${{{}}}".format(env_var)}
-
-    # Now append a new line  with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
-    with open(envFile, 'a') as f:
-        f.write("{}={}\n".format(env_var, port))
-
-    # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
-    # where the outer {{ }} will be replaced by {} in the returned string
-    return {"port": port, "env_var": "${{{}}}".format(env_var)}
-
-
-def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
-    lock = FileLock("citadel_registry_lock", dir="/tmp")
-    lock.acquire()
-    registryFile = path.join(nodeRoot, "apps", "registry.json")
-    registry: list = []
-    if path.isfile(registryFile):
-        with open(registryFile, 'r') as f:
-            registry = json.load(f)
-    else:
-        raise Exception("Registry file not found")
-
-    mainContainer = getMainContainer(app)
-
-    portDetails = assignPort(mainContainer, app.metadata.id, path.join(
-        nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
-    containerPort = portDetails['port']
-    portAsEnvVar = portDetails['env_var']
-    portToAppend = portAsEnvVar
-
-    mainPort = False
-
-    if mainContainer.port:
-        portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port)
-        mainPort = mainContainer.port
-        del mainContainer.port
-    else:
-        portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar)
-
-    if mainContainer.ports:
-        mainContainer.ports.append(portToAppend)
-        # Set the main port to the first port in the list, if it contains a :, it's the port after the :
-        # If it doesn't contain a :, it's the port itself
-        if mainPort == False:
-            mainPort = mainContainer.ports[0]
-            if mainPort.find(":") != -1:
-                mainPort = mainPort.split(":")[1]
-    else:
-        mainContainer.ports = [portToAppend]
-        if mainPort == False:
-            mainPort = portDetails['port']
-
-    if mainContainer.network_mode != "host":
-        mainContainer = assignIp(mainContainer, app.metadata.id)
-
-    # Also set the port in metadata
-    app.metadata.port = int(containerPort)
-    if mainPort:
-        app.metadata.internalPort = int(mainPort)
-    else:
-        app.metadata.internalPort = int(containerPort)
-
-    for registryApp in registry:
-        if registryApp['id'] == app.metadata.id:
-            registry[registry.index(registryApp)]['port'] = int(containerPort)
-            registry[registry.index(registryApp)]['internalPort'] = app.metadata.internalPort
-            break
-
-    with open(registryFile, 'w') as f:
-        json.dump(registry, f, indent=4, sort_keys=True)
-    lock.release()
-    return app

app/lib/composegenerator/v2/types.py

@@ -1,156 +0,0 @@
-from typing import Union, List
-from dataclasses import dataclass, field
-from dacite import from_dict
-
-@dataclass
-class Metadata:
-    id: str
-    name: str
-    version: str
-    category: str
-    tagline: str
-    description: str
-    developer: str
-    website: str
-    repo: str
-    support: str
-    gallery: List[str] = field(default_factory=list)
-    dependencies: List[str] = field(default_factory=list)
-    updateContainer: Union[str, Union[list, None]] = field(default_factory=list)
-    path: str = ""
-    defaultPassword: str = ""
-    torOnly: bool = False
-    lightningImplementation: Union[str, None] = None
-    # Added automatically later
-    port: int = 0
-    internalPort: int = 0
-
-@dataclass
-class Container:
-    name: str
-    image: str
-    permissions: list = field(default_factory=list)
-    ports: list = field(default_factory=list)
-    port: Union[int, None] = None
-    environment: Union[dict, None] = None
-    data: list = field(default_factory=list)
-    user: Union[str, None] = None
-    stop_grace_period: str = '1m'
-    depends_on: list = field(default_factory=list)
-    entrypoint: Union[List[str], str] = field(default_factory=list)
-    bitcoin_mount_dir: Union[str, None] = None
-    lnd_mount_dir: Union[str, None] = None
-    c_lightning_mount_dir: Union[str, None] = None
-    command: Union[List[str], str] = field(default_factory=list)
-    init: Union[bool, None] = None
-    stop_signal: Union[str, None] = None
-    noNetwork: Union[bool, None] = None
-    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
-    environment_allow: list = field(default_factory=list)
-    network_mode: Union[str, None] = None
-    # Only added later
-    volumes: list = field(default_factory=list)
-    restart: Union[str, None] = None
-
-@dataclass
-class App:
-    version: Union[str, int]
-    metadata: Metadata
-    containers: List[Container]
-
-# Generate an app instance from an app dict
-def generateApp(appDict):
-    return from_dict(data_class=App, data=appDict)
-
-@dataclass
-class Network:
-    ipv4_address: Union[str, None] = None
-
-@dataclass
-class NetworkConfig:
-    default: Network
-
-# After converting data dir and defining volumes, stage 2
-@dataclass
-class ContainerStage2:
-    id: str
-    name: str
-    image: str
-    permissions: List[str] = field(default_factory=list)
-    ports: list = field(default_factory=list)
-    environment: Union[dict, None] = None
-    user: Union[str, None] = None
-    stop_grace_period: str = '1m'
-    depends_on: List[str] = field(default_factory=list)
-    entrypoint: Union[List[str], str] = field(default_factory=list)
-    command: Union[List[str], str] = field(default_factory=list)
-    init: Union[bool, None] = None
-    stop_signal: Union[str, None] = None
-    noNetwork: Union[bool, None] = None
-    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
-    volumes: List[str] = field(default_factory=list)
-    networks: NetworkConfig = field(default_factory=NetworkConfig)
-    restart: Union[str, None] = None
-    network_mode: Union[str, None] = None
-
-@dataclass
-class AppStage2:
-    version: Union[str, int]
-    metadata: Metadata
-    containers: List[ContainerStage2]
-
-@dataclass
-class MetadataStage3:
-    id: str
-    name: str
-    version: str
-    category: str
-    tagline: str
-    description: str
-    developer: str
-    website: str
-    dependencies: List[str]
-    repo: str
-    support: str
-    gallery: List[str]
-    updateContainer: Union[str, Union[list, None]] = field(default_factory=list)
-    path: str = ""
-    defaultPassword: str = ""
-    torOnly: bool = False
-    lightningImplementation: Union[str, None] = None
-    # Added automatically later
-    port: int = 0
-    internalPort: int = 0
-
-@dataclass
-class AppStage3:
-    version: Union[str, int]
-    metadata: MetadataStage3
-    containers: List[ContainerStage2]
-
-@dataclass
-class ContainerStage4:
-    id: str
-    name: str
-    image: str
-    ports: list = field(default_factory=list)
-    environment: Union[dict, None] = None
-    user: Union[str, None] = None
-    stop_grace_period: str = '1m'
-    depends_on: List[str] = field(default_factory=list)
-    entrypoint: Union[List[str], str] = field(default_factory=list)
-    command: Union[List[str], str] = field(default_factory=list)
-    init: Union[bool, None] = None
-    stop_signal: Union[str, None] = None
-    noNetwork: Union[bool, None] = None
-    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
-    volumes: List[str] = field(default_factory=list)
-    networks: NetworkConfig = field(default_factory=NetworkConfig)
-    restart: Union[str, None] = None
-    network_mode: Union[str, None] = None
-
-@dataclass
-class AppStage4:
-    version: Union[str, int]
-    metadata: MetadataStage3
-    services: List[ContainerStage4]

app/lib/manage.py

@@ -30,7 +30,6 @@ except Exception:
     print("Continuing anyway, but some features won't be available,")
     print("for example checking for app updates")
 
-from lib.composegenerator.v2.generate import createComposeConfigFromV2
 from lib.validate import findAndValidateApps
 from lib.metadata import getAppRegistry
 from lib.entropy import deriveEntropy
@@ -172,12 +171,7 @@ def update(verbose: bool = False):
                 thread.start()
                 threads.append(thread)
             else:
-                appCompose = getApp(appDefinition, app)
+                raise Exception("Error: Unsupported version of app.yml")
-                with open(composeFile, "w") as f:
-                    if appCompose:
-                        f.write(yaml.dump(appCompose, sort_keys=False))
-                        if verbose:
-                            print("Wrote " + app + " to " + composeFile)
         except Exception as err:
             print("Failed to convert app {}".format(app))
             print(traceback.format_exc())
@@ -241,19 +235,6 @@ def stopInstalled():
         threads.append(thread)
     joinThreads(threads)
 
-# Loads an app.yml and converts it to a docker-compose.yml
-def getApp(app, appId: str):
-    if not "metadata" in app:
-        raise Exception("Error: Could not find metadata in " + appId)
-    app["metadata"]["id"] = appId
-
-    if 'version' in app and str(app['version']) == "2":
-        print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId))
-        return createComposeConfigFromV2(app, nodeRoot)
-    else:
-        raise Exception("Error: Unsupported version of app.yml")
-
-
 def compose(app, arguments):
     if not os.path.isdir(os.path.join(appsDir, app)):
         print("Warning: App {} doesn't exist on Citadel".format(app))

app/lib/metadata.py

@@ -6,8 +6,7 @@ import os
 import yaml
 import traceback
 
-from lib.composegenerator.shared.networking import assignIp, assignIpV4
+from lib.citadelutils import parse_dotenv
-from lib.composegenerator.v2.types import Container
 from lib.entropy import deriveEntropy
 from dacite import from_dict
 
@@ -15,6 +14,59 @@ appPorts = {}
 appPortMap = {}
 disabledApps = []
 
+def assignIpV4(appId: str, containerName: str):
+    scriptDir = path.dirname(path.realpath(__file__))
+    nodeRoot = path.join(scriptDir, "..", "..", "..", "..")
+    networkingFile = path.join(nodeRoot, "apps", "networking.json")
+    envFile = path.join(nodeRoot, ".env")
+    cleanContainerName = containerName.strip()
+    # If the name still contains a newline, throw an error
+    if cleanContainerName.find("\n") != -1:
+        raise Exception("Newline in container name")
+    env_var = "APP_{}_{}_IP".format(
+        appId.upper().replace("-", "_"),
+        cleanContainerName.upper().replace("-", "_")
+    )
+    # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
+    # can be used
+    usedIps = []
+    networkingData = {}
+    if path.isfile(networkingFile):
+        with open(networkingFile, 'r') as f:
+            networkingData = json.load(f)
+
+    if 'ip_addresses' in networkingData:
+        usedIps = list(networkingData['ip_addresses'].values())
+    else:
+        networkingData['ip_addresses'] = {}
+    # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
+    # If the IP is already in use, it will be tried again until it's not in use
+    # If it's not in use, it will be added to the usedIps list and written to the usedIpFile
+    # If the usedIpsFile contains all IPs between  10.21.21.20 and  10.21.21.255 (inclusive),
+    # Throw an error, because no more IPs can be used
+    if len(usedIps) == 235:
+        raise Exception("No more IPs can be used")
+
+    if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']:
+        ip = networkingData['ip_addresses']["{}-{}".format(
+            appId, cleanContainerName)]
+    else:
+        while True:
+            ip = "10.21.21." + str(random.randint(20, 255))
+            if ip not in usedIps:
+                networkingData['ip_addresses']["{}-{}".format(
+                    appId, cleanContainerName)] = ip
+                break
+
+    dotEnv = parse_dotenv(envFile)
+    if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
+        return
+
+    with open(envFile, 'a') as f:
+        f.write("{}={}\n".format(env_var, ip))
+    with open(networkingFile, 'w') as f:
+        json.dump(networkingData, f)
+
 def appPortsToMap():
     for port in appPorts:
         appId = appPorts[port]["app"]
@@ -64,9 +116,7 @@ def getAppRegistry(apps, app_path, portCache):
                         virtual_apps[implements] = []
                     virtual_apps[implements].append(app)
                 app_metadata.append(metadata)
-                if version == 2:
+                if version == 3:
-                    getPortsV2App(app_yml, app)
-                elif version == 3:
                     getPortsV3App(app_yml, app)
                 elif version == 4:
                     getPortsV4App(app_yml, app)
@@ -150,16 +200,6 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna
                         "dynamic": isDynamic,
                     }
 
-def getPortsV2App(app, appId):
-    for appContainer in app["containers"]:
-        if "port" in appContainer:
-            validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0)
-        if "ports" in appContainer:
-            for port in appContainer["ports"]:
-                realPort = int(str(port).split(":")[0])
-                validatePort(appContainer["name"], appContainer, realPort, appId, 2)
-
-
 def getPortsV3App(app, appId):
     for appContainer in app["containers"]:
         containerAsDataClass = from_dict(data_class=Container, data=appContainer)

app/lib/validate.py

@@ -11,27 +11,16 @@ import traceback
 scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
 nodeRoot = os.path.join(scriptDir, "..")
 
-with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f:
-    schemaVersion2 = yaml.safe_load(f)
-
 with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file: 
   dependencies = yaml.safe_load(file)
 
 # Validates app data
 # Returns true if valid, false otherwise
 def validateApp(app: dict):
-    if 'version' in app and str(app['version']) == "2":
+    if 'version' in app and str(app['version']) == "3":
-        try:
-            validate(app, schemaVersion2)
-            return True
-        # Catch and log any errors, and return false
-        except Exception as e:
-            print(traceback.format_exc())
-            return False
-    elif 'version' in app and str(app['version']) == "3":
         # The app-cli does this validation now
         return True
-    elif 'version' not in app and 'citadel_version' not in app:
+    elif 'citadel_version' not in app:
         print("Unsupported app version")
         return False
     else: