diff --git a/app-standard-v1.json b/app-standard-v1.json deleted file mode 100644 index 0ecb458..0000000 --- a/app-standard-v1.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Citadel app.yml v1", - "description": "The first draft of Citadel's app.yml format", - "type": "object", - "properties": { - "version": { - "type": [ - "string", - "number" - ], - "description": "The version of the app.yml format you're using." - }, - "metadata": { - "type": "object", - "properties": { - "name": { - "description": "Displayed name of the app", - "type": "string" - }, - "version": { - "description": "Displayed version for the app", - "type": "string" - }, - "category": { - "description": "The category you'd put the app in", - "type": "string" - }, - "tagline": { - "description": "A clever tagline", - "type": "string" - }, - "description": { - "description": "A longer description of the app", - "type": "string" - }, - "developer": { - "description": "The awesome people behind the app", - "type": "string" - }, - "website": { - "description": "Displayed version for the app", - "type": "string" - }, - "dependencies": { - "description": "The services the app depends on", - "type": "array", - "items": { - "type": "string" - } - }, - "repo": { - "description": "The development repository for your app", - "type": "string" - }, - "support": { - "description": "A link to the app support wiki/chat/...", - "type": "string" - }, - "gallery": { - "type": "array", - "description": "URLs or paths in the runcitadel/app-images/[app-name] folder with app images", - "items": { - "type": "string" - } - }, - "path": { - "description": "The path of the app's visible site the open button should open", - "type": "string" - }, - "defaultPassword": { - "description": "The app's default password", - "type": "string" - }, - "torOnly": { - "description": "Whether the app is only available over tor", - "type": "boolean" - }, - "mainContainer": { - "type": "string", - "description": "The name of the main container for the app. If set, IP, port, and hidden service will be assigned to it automatically." - }, - "updateContainer": { - "type": "string", - "description": "The container the developer system should automatically update." - } - }, - "required": [ - "name", - "version", - "category", - "tagline", - "description", - "developer", - "website", - "repo", - "support", - "gallery" - ], - "additionalProperties": false - }, - "containers": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "image": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "lnd", - "bitcoind", - "electrum", - "root", - "hw" - ] - } - }, - "ports": { - "type": "array", - "items": { - "type": [ - "string", - "number" - ] - } - }, - "port": { - "type": "number", - "description": "If this is the main container, the port inside the container which will be exposed to the outside as the port specified in metadata." - }, - "environment": { - "type": "object" - }, - "data": { - "type": "array", - "description": "An array of at directories in the container the app stores its data in. Can be empty. Please only list top-level directories.", - "items": { - "type": "string" - } - }, - "user": { - "type": "string", - "description": "The user the container should run as" - }, - "stop_grace_period": { - "type": "string", - "description": "The grace period for stopping the container. Defaults to 1 minute." - }, - "depends_on": { - "type": "array", - "description": "The services the container depends on" - }, - "entrypoint": { - "type": [ - "string", - "array" - ], - "description": "The entrypoint for the container" - }, - "bitcoin_mount_dir": { - "type": "string", - "description": "Where to mount the bitcoin dir" - }, - "command": { - "type": [ - "string", - "array" - ], - "description": "The command for the container" - }, - "init": { - "type": "boolean", - "description": "Whether the container should be run with init" - }, - "stop_signal": { - "type": "string", - "description": "The signal to send to the container when stopping" - }, - "noNetwork": { - "type": "boolean", - "description": "Set this to true if the container shouldn't get an IP & port exposed." - }, - "needsHiddenService": { - "type": "boolean", - "description": "Set this to true if the container should be assigned a hidden service even if it's not the main container." - }, - "hiddenServicePort": { - "type": "number", - "description": "Set this to a port if your container exposes multiple ports, but only one should be a hidden service." - }, - "hiddenServicePorts": { - "type": "object", - "description": "Set this to a map of service names to hidden service ports if your container exposes multiple ports, and all of them should be hidden services.", - "patternProperties": { - "^[a-zA-Z0-9_]+$": { - "type": [ - "number", - "array" - ] - } - } - }, - "restart": { - "type": "string", - "description": "When the container should restart. Can be 'always' or 'on-failure'." - } - }, - "additionalProperties": false, - "required": [ - "name", - "image" - ] - }, - "additionalProperties": false - } - }, - "required": [ - "metadata", - "containers" - ], - "additionalProperties": false -} diff --git a/app-standard-v1.json.license b/app-standard-v1.json.license deleted file mode 100644 index f0626d9..0000000 --- a/app-standard-v1.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2021 Citadel and contributors - -SPDX-License-Identifier: GPL-3.0-or-later \ No newline at end of file diff --git a/app/app-standard-v1.yml b/app/app-standard-v1.yml deleted file mode 100644 index 3000a25..0000000 --- a/app/app-standard-v1.yml +++ /dev/null @@ -1,194 +0,0 @@ -# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema -$schema: https://json-schema.org/draft/2020-12/schema - - -title: Citadel app.yml v1 -description: The first draft of Citadel's app.yml format -type: object - -properties: - version: - type: - - string - - number - description: The version of the app.yml format you're using. - - metadata: - type: object - properties: - name: - description: Displayed name of the app - type: string - version: - description: Displayed version for the app - type: string - category: - description: The category you'd put the app in - type: string - tagline: - description: A clever tagline - type: string - description: - description: A longer description of the app - type: string - developer: - description: The awesome people behind the app - type: string - website: - description: Displayed version for the app - type: string - dependencies: - description: The services the app depends on - type: array - items: - type: string - repo: - description: The development repository for your app - type: string - support: - description: A link to the app support wiki/chat/... - type: string - gallery: - type: array - description: >- - URLs or paths in the runcitadel/app-images/[app-name] folder with app - images - items: - type: string - path: - description: The path of the app's visible site the open button should open - type: string - defaultPassword: - description: The app's default password - type: string - torOnly: - description: Whether the app is only available over tor - type: boolean - mainContainer: - type: string - description: >- - The name of the main container for the app. If set, IP, port, and - hidden service will be assigned to it automatically. - updateContainer: - type: string - description: The container the developer system should automatically update. - required: - - name - - version - - category - - tagline - - description - - developer - - website - - repo - - support - - gallery - additionalProperties: false - - containers: - type: array - items: - type: object - properties: - name: - type: string - image: - type: string - permissions: - type: array - items: - type: string - enum: - - lnd - - bitcoind - - electrum - - root - - hw - ports: - type: array - items: - type: - - string - - number - port: - type: number - description: >- - If this is the main container, the port inside the container which - will be exposed to the outside as the port specified in metadata. - environment: - type: object - data: - type: array - description: >- - An array of at directories in the container the app stores its data - in. Can be empty. Please only list top-level directories. - items: - type: string - user: - type: string - description: The user the container should run as - stop_grace_period: - type: string - description: The grace period for stopping the container. Defaults to 1 minute. - depends_on: - type: array - description: The services the container depends on - entrypoint: - type: - - string - - array - description: The entrypoint for the container - bitcoin_mount_dir: - type: string - description: Where to mount the bitcoin dir - command: - type: - - string - - array - description: The command for the container - init: - type: boolean - description: Whether the container should be run with init - stop_signal: - type: string - description: The signal to send to the container when stopping - noNetwork: - type: boolean - description: >- - Set this to true if the container shouldn't get an IP & port - exposed. - needsHiddenService: - type: boolean - description: >- - Set this to true if the container should be assigned a hidden - service even if it's not the main container. - hiddenServicePort: - type: number - description: >- - Set this to a port if your container exposes multiple ports, but - only one should be a hidden service. - hiddenServicePorts: - type: object - description: >- - Set this to a map of service names to hidden service ports if your - container exposes multiple ports, and all of them should be hidden - services. - patternProperties: - ^[a-zA-Z0-9_]+$: - type: - - number - - array - restart: - type: string - description: When the container should restart. Can be 'always' or 'on-failure'. - additionalProperties: false - required: - - name - - image - additionalProperties: false - -required: - - metadata - - containers - -additionalProperties: false diff --git a/app/lib/citadelutils.py b/app/lib/citadelutils.py index e0b044e..bf8d893 100644 --- a/app/lib/citadelutils.py +++ b/app/lib/citadelutils.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: GPL-3.0-or-later import re +import fcntl +import os # Helper functions @@ -77,3 +79,27 @@ def classToDict(theClass): obj[key] = classToDict(value) return obj +class FileLock: + """Implements a file-based lock using flock(2). + The lock file is saved in directory dir with name lock_name. + dir is the current directory by default. + """ + + def __init__(self, lock_name, dir="."): + self.lock_file = open(os.path.join(dir, lock_name), "w") + + def acquire(self, blocking=True): + """Acquire the lock. + If the lock is not already acquired, return None. If the lock is + acquired and blocking is True, block until the lock is released. If + the lock is acquired and blocking is False, raise an IOError. + """ + ops = fcntl.LOCK_EX + if not blocking: + ops |= fcntl.LOCK_NB + fcntl.flock(self.lock_file, ops) + + def release(self): + """Release the lock. Return None even if lock not currently acquired""" + fcntl.flock(self.lock_file, fcntl.LOCK_UN) + \ No newline at end of file diff --git a/app/lib/composegenerator/shared/env.py b/app/lib/composegenerator/shared/env.py index 69b4318..fb12f1a 100644 --- a/app/lib/composegenerator/shared/env.py +++ b/app/lib/composegenerator/shared/env.py @@ -4,7 +4,7 @@ import re from typing import Union -from lib.composegenerator.v1.types import App +from lib.composegenerator.v2.types import App from lib.composegenerator.shared.const import always_allowed_env from lib.citadelutils import checkArrayContainsAllElements, getEnvVars diff --git a/app/lib/composegenerator/shared/main.py b/app/lib/composegenerator/shared/main.py index 4cc14df..ff9922d 100644 --- a/app/lib/composegenerator/shared/main.py +++ b/app/lib/composegenerator/shared/main.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # Main functions -from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container +from lib.composegenerator.v2.types import App, AppStage3 from lib.composegenerator.shared.const import permissions @@ -27,28 +27,3 @@ def convertContainersToServices(app: AppStage3) -> AppStage3: del app.containers app.services = services return app - -# Converts the data of every container in app.containers to a volume, which is then added to the app -def convertDataDirToVolume(app: App) -> AppStage2: - for container in app.containers: - # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes - # Also, a datadir shouldn't start with a / - for dataDir in container.data: - if dataDir.find("..") == -1 and dataDir[0] != "/": - container.volumes.append( - '${APP_DATA_DIR}/' + dataDir) - else: - print("Data dir " + dataDir + - " contains invalid characters") - del container.data - if container.bitcoin_mount_dir != None: - if not 'bitcoind' in container.permissions: - print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name)) - # Skip this container - continue - # Also skip the container if container.bitcoin_mount_dir contains a : - if container.bitcoin_mount_dir.find(":") == -1: - container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir) - del container.bitcoin_mount_dir - - return app diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py new file mode 100644 index 0000000..7c1c397 --- /dev/null +++ b/app/lib/composegenerator/shared/networking.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +import json +from os import path +import random +from lib.composegenerator.shared.utils.networking import getContainerHiddenService +from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container +from lib.citadelutils import parse_dotenv +from dacite import from_dict + +def getMainContainer(app: App) -> Container: + if len(app.containers) == 1: + return app.containers[0] + else: + for container in app.containers: + # Main is recommended, support web for easier porting from Umbrel + if container.name == 'main' or container.name == 'web': + return container + # Fallback to first container + return app.containers[0] + +def assignIpV4(appId: str, containerName: str): + scriptDir = path.dirname(path.realpath(__file__)) + nodeRoot = path.join(scriptDir, "..", "..", "..", "..") + networkingFile = path.join(nodeRoot, "apps", "networking.json") + envFile = path.join(nodeRoot, ".env") + cleanContainerName = containerName.strip() + # If the name still contains a newline, throw an error + if cleanContainerName.find("\n") != -1: + raise Exception("Newline in container name") + env_var = "APP_{}_{}_IP".format( + appId.upper().replace("-", "_"), + cleanContainerName.upper().replace("-", "_") + ) + # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP + # can be used + usedIps = [] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + + if 'ip_addresses' in networkingData: + usedIps = list(networkingData['ip_addresses'].values()) + else: + networkingData['ip_addresses'] = {} + # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container + # If the IP is already in use, it will be tried again until it's not in use + # If it's not in use, it will be added to the usedIps list and written to the usedIpFile + # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), + # Throw an error, because no more IPs can be used + if len(usedIps) == 235: + raise Exception("No more IPs can be used") + + if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']: + ip = networkingData['ip_addresses']["{}-{}".format( + appId, cleanContainerName)] + else: + while True: + ip = "10.21.21." + str(random.randint(20, 255)) + if ip not in usedIps: + networkingData['ip_addresses']["{}-{}".format( + appId, cleanContainerName)] = ip + break + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): + return + + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, ip)) + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + +def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1: + raise Exception("Newline in container name") + env_var = "APP_{}_{}_IP".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP + # can be used + usedIps = [] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + + if 'ip_addresses' in networkingData: + usedIps = list(networkingData['ip_addresses'].values()) + else: + networkingData['ip_addresses'] = {} + # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container + # If the IP is already in use, it will be tried again until it's not in use + # If it's not in use, it will be added to the usedIps list and written to the usedIpFile + # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), + # Throw an error, because no more IPs can be used + if len(usedIps) == 235: + raise Exception("No more IPs can be used") + + if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: + ip = networkingData['ip_addresses']["{}-{}".format( + appId, container.name)] + else: + while True: + ip = "10.21.21." + str(random.randint(20, 255)) + if ip not in usedIps: + networkingData['ip_addresses']["{}-{}".format( + appId, container.name)] = ip + break + container.networks = from_dict(data_class=NetworkConfig, data={'default': { + 'ipv4_address': "$" + env_var}}) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): + return container + + # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, ip)) + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + return container + +def configureIps(app: AppStage2, networkingFile: str, envFile: str): + for container in app.containers: + if container.network_mode and container.network_mode == "host": + continue + if container.noNetwork: + # Check if port is defined for the container + if container.port: + raise Exception("Port defined for container without network") + if getMainContainer(app).name == container.name: + raise Exception("Main container without network") + # Skip this iteration of the loop + continue + + container = assignIp(container, app.metadata.id, + networkingFile, envFile) + + return app + +def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3: + dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) + hiddenServices = "" + + mainContainer = getMainContainer(app) + + for container in app.containers: + if container.network_mode and container.network_mode == "host": + continue + env_var = "APP_{}_{}_IP".format( + app.metadata.id.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + hiddenServices += getContainerHiddenService( + app.metadata, container, dotEnv[env_var], container.name == mainContainer.name) + if container.hiddenServicePorts: + del container.hiddenServicePorts + + torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] + torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] + with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: + f.write(hiddenServices) + return app diff --git a/app/lib/composegenerator/v2/utils/networking.py b/app/lib/composegenerator/shared/utils/networking.py similarity index 100% rename from app/lib/composegenerator/v2/utils/networking.py rename to app/lib/composegenerator/shared/utils/networking.py diff --git a/app/lib/composegenerator/v1/generate.py b/app/lib/composegenerator/v1/generate.py deleted file mode 100644 index fa01fd4..0000000 --- a/app/lib/composegenerator/v1/generate.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from lib.composegenerator.v1.types import App, AppStage4, generateApp -from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices -from lib.composegenerator.shared.env import validateEnv -from lib.citadelutils import classToDict -import os - -def createComposeConfigFromV1(app: dict, nodeRoot: str): - envFile = os.path.join(nodeRoot, ".env") - networkingFile = os.path.join(nodeRoot, "apps", "networking.json") - - newApp: App = generateApp(app) - newApp = convertContainerPermissions(newApp) - validateEnv(newApp) - newApp = convertDataDirToVolume(newApp) - newApp = configureIps(newApp, networkingFile, envFile) - newApp = configureMainPort(newApp, nodeRoot) - configureHiddenServices(newApp, nodeRoot) - finalConfig: AppStage4 = convertContainersToServices(newApp) - newApp = classToDict(finalConfig) - del newApp['metadata'] - if "version" in newApp: - del newApp["version"] - # Set version to 3.8 (current compose file version) - newApp = {'version': '3.8', **newApp} - return newApp diff --git a/app/lib/composegenerator/v1/networking.py b/app/lib/composegenerator/v1/networking.py deleted file mode 100644 index 68d1dc1..0000000 --- a/app/lib/composegenerator/v1/networking.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from dacite import from_dict -from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig -from lib.citadelutils import parse_dotenv -import json -from os import path -import random -from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService - - -def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2: - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1: - raise Exception("Newline in container name") - env_var = "APP_{}_{}_IP".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP - # can be used - usedIps = [] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - - if 'ip_addresses' in networkingData: - usedIps = list(networkingData['ip_addresses'].values()) - else: - networkingData['ip_addresses'] = {} - # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container - # If the IP is already in use, it will be tried again until it's not in use - # If it's not in use, it will be added to the usedIps list and written to the usedIpFile - # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), - # Throw an error, because no more IPs can be used - if len(usedIps) == 235: - raise Exception("No more IPs can be used") - - if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']: - ip = networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] - else: - while True: - ip = "10.21.21." + str(random.randint(20, 255)) - if ip not in usedIps: - networkingData['ip_addresses']["{}-{}".format( - appId, container.name)] = ip - break - container.networks = from_dict(data_class=NetworkConfig, data={'default': { - 'ipv4_address': "$" + env_var}}) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(ip): - return container - - # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, ip)) - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - return container - - -def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1 or container.name.find(" ") != -1: - raise Exception("Newline or space in container name") - - env_var = "APP_{}_{}_PORT".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - - port = getFreePort(networkingFile, appId) - - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(port): - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, port)) - - # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} - # where the outer {{ }} will be replaced by {} in the returned string - return {"port": port, "env_var": "${{{}}}".format(env_var)} - - -def getMainContainer(app: dict): - if len(app.containers) == 1: - return app.containers[0] - else: - if not app.metadata.mainContainer: - app.metadata.mainContainer = 'main' - for container in app.containers: - if container.name == app.metadata.mainContainer: - return container - raise Exception( - "No main container found for app {}".format(app.metadata.name)) - - -def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: - registryFile = path.join(nodeRoot, "apps", "registry.json") - registry: list = [] - if path.isfile(registryFile): - with open(registryFile, 'r') as f: - registry = json.load(f) - else: - raise Exception("Registry file not found") - - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - - mainContainer = getMainContainer(app) - - portDetails = assignPort(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - containerPort = portDetails['port'] - portAsEnvVar = portDetails['env_var'] - portToAppend = portAsEnvVar - - mainPort = False - - if mainContainer.port: - portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port) - mainPort = mainContainer.port - del mainContainer.port - else: - portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar) - - if mainContainer.ports: - mainContainer.ports.append(portToAppend) - # Set the main port to the first port in the list, if it contains a :, it's the port after the : - # If it doesn't contain a :, it's the port itself - if mainPort == False: - mainPort = mainContainer.ports[0] - if mainPort.find(":") != -1: - mainPort = mainPort.split(":")[1] - else: - mainContainer.ports = [portToAppend] - if mainPort == False: - mainPort = portDetails['port'] - - mainContainer = assignIp(mainContainer, app.metadata.id, path.join( - nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) - - # If the IP wasn't in dotenv before, now it should be - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - - containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace( - "-", "_"), mainContainer.name.upper().replace("-", "_"))] - - hiddenservice = getHiddenService( - app.metadata.name, app.metadata.id, containerIP, mainPort) - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenservice) - - # Also set the port in metadata - app.metadata.port = int(containerPort) - - for registryApp in registry: - if registryApp['id'] == app.metadata.id: - registry[registry.index(registryApp)]['port'] = int(containerPort) - break - - with open(registryFile, 'w') as f: - json.dump(registry, f, indent=4, sort_keys=True) - - return app - - -def configureIps(app: AppStage2, networkingFile: str, envFile: str): - for container in app.containers: - if container.noNetwork: - # Check if port is defined for the container - if container.port: - raise Exception("Port defined for container without network") - if app.metadata.mainContainer == container.name: - raise Exception("Main container without network") - # Skip this iteration of the loop - continue - - container = assignIp(container, app.metadata.id, - networkingFile, envFile) - - return app - - -def configureHiddenServices(app: dict, nodeRoot: str) -> None: - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - hiddenServices = "" - - if len(app.containers) == 1: - mainContainer = app.containers[0] - else: - mainContainer = None - if app.metadata.mainContainer == None: - app.metadata.mainContainer = 'main' - for container in app.containers: - if container.name == app.metadata.mainContainer: - mainContainer = container - break - if mainContainer is None: - raise Exception("No main container found") - - for container in app.containers: - env_var = "APP_{}_{}_IP".format( - app.metadata.id.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - hiddenServices += getContainerHiddenService( - app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name) - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenServices) diff --git a/app/lib/composegenerator/v1/types.py b/app/lib/composegenerator/v1/types.py deleted file mode 100644 index c9556be..0000000 --- a/app/lib/composegenerator/v1/types.py +++ /dev/null @@ -1,151 +0,0 @@ -from typing import Union, List -from dataclasses import dataclass, field -from dacite import from_dict - -@dataclass -class Metadata: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - repo: str - support: str - gallery: List[str] = field(default_factory=list) - dependencies: List[str] = field(default_factory=list) - mainContainer: Union[str, None] = None - updateContainer: Union[str, None] = None - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - -@dataclass -class Container: - name: str - image: str - permissions: list = field(default_factory=list) - ports: list = field(default_factory=list) - port: Union[int, None] = None - environment: Union[dict, None] = None - data: list = field(default_factory=list) - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: list = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - bitcoin_mount_dir: Union[str, None] = None - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - environment_allow: list = field(default_factory=list) - # Only added later - volumes: list = field(default_factory=list) - restart: Union[str, None] = None - -@dataclass -class App: - version: Union[str, int] - metadata: Metadata - containers: List[Container] - -# Generate an app instance from an app dict -def generateApp(appDict): - return from_dict(data_class=App, data=appDict) - -@dataclass -class Network: - ipv4_address: Union[str, None] = None - -@dataclass -class NetworkConfig: - default: Network - -# After converting data dir and defining volumes, stage 2 -@dataclass -class ContainerStage2: - id: str - name: str - image: str - permissions: List[str] = field(default_factory=list) - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - -@dataclass -class AppStage2: - version: Union[str, int] - metadata: Metadata - containers: List[ContainerStage2] - -@dataclass -class MetadataStage3: - id: str - name: str - version: str - category: str - tagline: str - description: str - developer: str - website: str - dependencies: List[str] - repo: str - support: str - gallery: List[str] - mainContainer: Union[str, None] = None - updateContainer: Union[str, None] = None - path: str = "" - defaultPassword: str = "" - torOnly: bool = False - -@dataclass -class AppStage3: - version: Union[str, int] - metadata: MetadataStage3 - containers: List[ContainerStage2] - -@dataclass -class ContainerStage4: - id: str - name: str - image: str - ports: list = field(default_factory=list) - environment: Union[dict, None] = None - user: Union[str, None] = None - stop_grace_period: str = '1m' - depends_on: List[str] = field(default_factory=list) - entrypoint: Union[List[str], str] = field(default_factory=list) - command: Union[List[str], str] = field(default_factory=list) - init: Union[bool, None] = None - stop_signal: Union[str, None] = None - noNetwork: Union[bool, None] = None - needsHiddenService: Union[bool, None] = None - hiddenServicePort: Union[int, None] = None - hiddenServicePorts: Union[dict, None] = None - volumes: List[str] = field(default_factory=list) - networks: NetworkConfig = field(default_factory=NetworkConfig) - restart: Union[str, None] = None - -@dataclass -class AppStage4: - version: Union[str, int] - metadata: MetadataStage3 - services: List[ContainerStage4] \ No newline at end of file diff --git a/app/lib/composegenerator/v1/utils/networking.py b/app/lib/composegenerator/v1/utils/networking.py deleted file mode 100644 index 2668aa7..0000000 --- a/app/lib/composegenerator/v1/utils/networking.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import json -import os -import random - -from lib.composegenerator.v1.types import Container - -def getFreePort(networkingFile: str, appId: str): - # Ports used currently in Citadel - usedPorts = [ - # Dashboard - 80, - # Sometimes used by nginx with some setups - 433, - # Dashboard SSL - 443, - # Bitcoin Core P2P - 8333, - # LND gRPC - 10009, - # LND REST - 8080, - # Electrum Server - 50001, - # Tor Proxy - 9050, - ] - networkingData = {} - if os.path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - if 'ports' in networkingData: - usedPorts += list(networkingData['ports'].values()) - else: - networkingData['ports'] = {} - - if appId in networkingData['ports']: - return networkingData['ports'][appId] - - while True: - port = str(random.randint(1024, 49151)) - if port not in usedPorts: - # Check if anyhing is listening on the specific port - if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: - networkingData['ports'][appId] = port - break - - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - - return port - - -def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str: - hiddenServices = ''' -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -'''.format(name, id) - for port in ports: - hiddenServices += 'HiddenServicePort {} {}:{}'.format( - port, internalIp, port) - hiddenServices += "\n" - return hiddenServices - - -def getHiddenServiceString(name: str, id: str, internalPort, internalIp: str, publicPort) -> str: - return ''' -# {} Hidden Service -HiddenServiceDir /var/lib/tor/app-{} -HiddenServicePort {} {}:{} - -'''.format(name, id, publicPort, internalIp, internalPort) - - -def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str: - return getHiddenServiceString(appName, appId, appPort, appIp, "80") - - -def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str: - if not container.needsHiddenService and not isMainContainer: - return "" - if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer: - print("Container {} for app {} isn't compatible with hidden service assignment".format( - container.name, appName)) - return "" - - if isMainContainer: - if not container.hiddenServicePorts: - return "" - # hiddenServicePorts is a map of hidden service name to port - # We need to generate a hidden service for each one - hiddenServices = "" - for name, port in container.hiddenServicePorts.items(): - if ".." in name: - print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName)) - - # If port is a list, use getHiddenServiceMultiPort - if isinstance(port, list): - hiddenServices += getHiddenServiceMultiPort("{} {}".format(appName, name), "{}-{}".format( - appId, name), containerIp, port) - else: - hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format( - appId, name), port, containerIp, port) - del container.hiddenServicePorts - return hiddenServices - - del container.needsHiddenService - if not container.port: - data = getHiddenServiceString(appName + container.name, "{}-{}".format( - appId, container.name), container.hiddenServicePort, containerIp, "80") - del container.hiddenServicePort - return data - else: - return getHiddenServiceString(appName + container.name, "{}-{}".format( - appId, container.name), container.port, containerIp, container.port) diff --git a/app/lib/composegenerator/v2/generate.py b/app/lib/composegenerator/v2/generate.py index dfcd38d..fbb08bd 100644 --- a/app/lib/composegenerator/v2/generate.py +++ b/app/lib/composegenerator/v2/generate.py @@ -3,15 +3,34 @@ # SPDX-License-Identifier: GPL-3.0-or-later from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp -from lib.composegenerator.v2.networking import configureHiddenServices, configureIps, configureMainPort -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices +from lib.composegenerator.v2.networking import configureMainPort +from lib.composegenerator.shared.networking import configureHiddenServices, configureIps +from lib.composegenerator.shared.main import convertContainerPermissions, convertContainersToServices from lib.composegenerator.shared.env import validateEnv from lib.citadelutils import classToDict import os def convertDataDirToVolumeGen2(app: App) -> AppStage2: - app = convertDataDirToVolume(app) for container in app.containers: + # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes + # Also, a datadir shouldn't start with a / + for dataDir in container.data: + if dataDir.find("..") == -1 and dataDir[0] != "/": + container.volumes.append( + '${APP_DATA_DIR}/' + dataDir) + else: + print("Data dir " + dataDir + + " contains invalid characters") + del container.data + if container.bitcoin_mount_dir != None: + if not 'bitcoind' in container.permissions: + print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name)) + # Skip this container + continue + # Also skip the container if container.bitcoin_mount_dir contains a : + if container.bitcoin_mount_dir.find(":") == -1: + container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir) + del container.bitcoin_mount_dir if container.lnd_mount_dir != None: if not 'lnd' in container.permissions: print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name)) diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py index ec5d4ad..e144867 100644 --- a/app/lib/composegenerator/v2/networking.py +++ b/app/lib/composegenerator/v2/networking.py @@ -2,14 +2,59 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container from lib.citadelutils import parse_dotenv +from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container import json from os import path +import os import random -from lib.composegenerator.v2.utils.networking import getContainerHiddenService -from lib.composegenerator.v1.networking import assignIp, assignPort +from lib.composegenerator.shared.networking import assignIp +from lib.citadelutils import FileLock +def getFreePort(networkingFile: str, appId: str): + # Ports used currently in Citadel + usedPorts = [ + # Dashboard + 80, + # Sometimes used by nginx with some setups + 433, + # Dashboard SSL + 443, + # Bitcoin Core P2P + 8333, + # LND gRPC + 10009, + # LND REST + 8080, + # Electrum Server + 50001, + # Tor Proxy + 9050, + ] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + if 'ports' in networkingData: + usedPorts += list(networkingData['ports'].values()) + else: + networkingData['ports'] = {} + + if appId in networkingData['ports']: + return networkingData['ports'][appId] + + while True: + port = str(random.randint(1024, 49151)) + if port not in usedPorts: + # Check if anyhing is listening on the specific port + if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: + networkingData['ports'][appId] = port + break + + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + + return port def getMainContainer(app: App) -> Container: if len(app.containers) == 1: @@ -22,8 +67,36 @@ def getMainContainer(app: App) -> Container: # Fallback to first container return app.containers[0] +def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1 or container.name.find(" ") != -1: + raise Exception("Newline or space in container name") + + env_var = "APP_{}_{}_PORT".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + + port = getFreePort(networkingFile, appId) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(port): + return {"port": port, "env_var": "${{{}}}".format(env_var)} + + # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, port)) + + # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} + # where the outer {{ }} will be replaced by {} in the returned string + return {"port": port, "env_var": "${{{}}}".format(env_var)} + def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: + lock = FileLock("citadel_registry_lock", dir="/tmp") + lock.acquire() registryFile = path.join(nodeRoot, "apps", "registry.json") registry: list = [] if path.isfile(registryFile): @@ -81,48 +154,5 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: with open(registryFile, 'w') as f: json.dump(registry, f, indent=4, sort_keys=True) - - return app - -def configureIps(app: AppStage2, networkingFile: str, envFile: str): - for container in app.containers: - if container.network_mode and container.network_mode == "host": - continue - if container.noNetwork: - # Check if port is defined for the container - if container.port: - raise Exception("Port defined for container without network") - if getMainContainer(app).name == container.name: - raise Exception("Main container without network") - # Skip this iteration of the loop - continue - - container = assignIp(container, app.metadata.id, - networkingFile, envFile) - - return app - - -def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3: - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - hiddenServices = "" - - mainContainer = getMainContainer(app) - - for container in app.containers: - if container.network_mode and container.network_mode == "host": - continue - env_var = "APP_{}_{}_IP".format( - app.metadata.id.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - hiddenServices += getContainerHiddenService( - app.metadata, container, dotEnv[env_var], container.name == mainContainer.name) - if container.hiddenServicePorts: - del container.hiddenServicePorts - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenServices) + lock.release() return app diff --git a/app/lib/composegenerator/v3/generate.py b/app/lib/composegenerator/v3/generate.py index 3e2789f..1164bb1 100644 --- a/app/lib/composegenerator/v3/generate.py +++ b/app/lib/composegenerator/v3/generate.py @@ -5,9 +5,9 @@ import os from lib.citadelutils import classToDict -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainersToServices +from lib.composegenerator.shared.main import convertContainersToServices from lib.composegenerator.shared.env import validateEnv -from lib.composegenerator.v2.networking import configureIps, configureHiddenServices +from lib.composegenerator.shared.networking import configureIps, configureHiddenServices from lib.composegenerator.v3.types import App, AppStage2, AppStage4, generateApp from lib.composegenerator.v3.networking import configureMainPort diff --git a/app/lib/composegenerator/v3/networking.py b/app/lib/composegenerator/v3/networking.py index 66d5a96..edb696c 100644 --- a/app/lib/composegenerator/v3/networking.py +++ b/app/lib/composegenerator/v3/networking.py @@ -2,13 +2,11 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from lib.composegenerator.v3.types import App, AppStage2, AppStage3, Container -from lib.citadelutils import parse_dotenv +from lib.composegenerator.v3.types import App, AppStage2, AppStage3 import json from os import path -import random -from lib.composegenerator.v1.networking import assignIp, assignPort - +from lib.composegenerator.shared.networking import assignIp +from lib.citadelutils import FileLock def getMainContainerIndex(app: App): if len(app.containers) == 1: @@ -27,6 +25,8 @@ def getMainContainerIndex(app: App): def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: + lock = FileLock("citadel_registry_lock", dir="/tmp") + lock.acquire() registryFile = path.join(nodeRoot, "apps", "registry.json") portsFile = path.join(nodeRoot, "apps", "ports.json") envFile = path.join(nodeRoot, ".env") @@ -101,8 +101,8 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: with open(registryFile, 'w') as f: json.dump(registry, f, indent=4, sort_keys=True) + lock.release() with open(envFile, 'a') as f: f.write("{}={}\n".format(portAsEnvVar, app.metadata.port)) return app - diff --git a/app/lib/manage.py b/app/lib/manage.py index 1ae8c0f..56f288d 100644 --- a/app/lib/manage.py +++ b/app/lib/manage.py @@ -10,7 +10,6 @@ import random from typing import List from sys import argv import os -import fcntl import requests import shutil import json @@ -28,36 +27,12 @@ except Exception: print("Continuing anyway, but some features won't be available,") print("for example checking for app updates") -from lib.composegenerator.v1.generate import createComposeConfigFromV1 from lib.composegenerator.v2.generate import createComposeConfigFromV2 from lib.composegenerator.v3.generate import createComposeConfigFromV3 from lib.validate import findAndValidateApps from lib.metadata import getAppRegistry from lib.entropy import deriveEntropy - -class FileLock: - """Implements a file-based lock using flock(2). - The lock file is saved in directory dir with name lock_name. - dir is the current directory by default. - """ - - def __init__(self, lock_name, dir="."): - self.lock_file = open(os.path.join(dir, lock_name), "w") - - def acquire(self, blocking=True): - """Acquire the lock. - If the lock is not already acquired, return None. If the lock is - acquired and blocking is True, block until the lock is released. If - the lock is acquired and blocking is False, raise an IOError. - """ - ops = fcntl.LOCK_EX - if not blocking: - ops |= fcntl.LOCK_NB - fcntl.flock(self.lock_file, ops) - - def release(self): - """Release the lock. Return None even if lock not currently acquired""" - fcntl.flock(self.lock_file, fcntl.LOCK_UN) +from lib.citadelutils import FileLock # For an array of threads, join them and wait for them to finish def joinThreads(threads: List[threading.Thread]): @@ -111,7 +86,7 @@ def handleAppV4(app): for registryApp in registry: if registryApp['id'] == app: - registry[registry.index(registryApp)]['port'] = resultYml["port"] + registry[registry.index(registryApp)]['port'] = mainPort break with open(registryFile, 'w') as f: @@ -138,12 +113,19 @@ def getAppYml(name): def update(verbose: bool = False): apps = findAndValidateApps(appsDir) + portCache = {} + try: + with open(os.path.join(appsDir, "ports.cache.json"), "w") as f: + portCache = json.load(f) + except Exception: pass # The compose generation process updates the registry, so we need to get it set up with the basics before that - registry = getAppRegistry(apps, appsDir) + registry = getAppRegistry(apps, appsDir, portCache) with open(os.path.join(appsDir, "registry.json"), "w") as f: json.dump(registry["metadata"], f, sort_keys=True) with open(os.path.join(appsDir, "ports.json"), "w") as f: json.dump(registry["ports"], f, sort_keys=True) + with open(os.path.join(appsDir, "ports.cache.json"), "w") as f: + json.dump(registry["portCache"], f, sort_keys=True) with open(os.path.join(appsDir, "virtual-apps.json"), "w") as f: json.dump(registry["virtual_apps"], f, sort_keys=True) print("Wrote registry to registry.json") @@ -234,17 +216,14 @@ def stopInstalled(): # Loads an app.yml and converts it to a docker-compose.yml def getApp(app, appId: str): if not "metadata" in app: - raise Exception("Error: Could not find metadata in " + appFile) + raise Exception("Error: Could not find metadata in " + appId) app["metadata"]["id"] = appId - if 'version' in app and str(app['version']) == "1": - print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId)) - return createComposeConfigFromV1(app, nodeRoot) - elif 'version' in app and str(app['version']) == "2": - print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId)) + if 'version' in app and str(app['version']) == "2": + print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId)) return createComposeConfigFromV2(app, nodeRoot) elif 'version' in app and str(app['version']) == "3": - print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.3.0".format(appId)) + print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId)) return createComposeConfigFromV3(app, nodeRoot) else: raise Exception("Error: Unsupported version of app.yml") diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 8f5bc11..d8679c8 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -6,12 +6,8 @@ import os import yaml import traceback -from lib.composegenerator.v2.networking import getMainContainer from lib.composegenerator.shared.networking import assignIpV4 from lib.entropy import deriveEntropy -from typing import List -import json -import random appPorts = {} appPortMap = {} @@ -37,9 +33,10 @@ def appPortsToMap(): # Also check the path and defaultPassword and set them to an empty string if they don't exist # In addition, set id on the metadata to the name of the app # Return a list of all app's metadata -def getAppRegistry(apps, app_path): +def getAppRegistry(apps, app_path, portCache): app_metadata = [] virtual_apps = {} + appPorts = portCache for app in apps: app_yml_path = os.path.join(app_path, app, 'app.yml') if os.path.isfile(app_yml_path): @@ -78,7 +75,8 @@ def getAppRegistry(apps, app_path): return { "virtual_apps": virtual_apps, "metadata": app_metadata, - "ports": appPortMap + "ports": appPortMap, + "portCache": appPorts, } citadelPorts = [ @@ -102,11 +100,11 @@ citadelPorts = [ lastPort = 3000 -def getNewPort(appPorts, appId): +def getNewPort(usedPorts, appId, containerName, allowExisting): lastPort2 = lastPort - while lastPort2 in appPorts.keys() or lastPort2 in citadelPorts: - if lastPort2 in appPorts.keys() and appPorts[lastPort2]["app"] == appId: - return lastPort2 + while lastPort2 in usedPorts.keys() or lastPort2 in citadelPorts: + if allowExisting and lastPort2 in usedPorts.keys() and usedPorts[lastPort2]["app"] == appId and usedPorts[lastPort2]["container"] == containerName: + break lastPort2 = lastPort2 + 1 return lastPort2 @@ -120,10 +118,10 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna "dynamic": isDynamic, } else: - if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != appContainer["name"]: - newPort = getNewPort(appPorts, appId) + if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != containerName: if port in appPorts and priority > appPorts[port]["priority"]: #print("Prioritizing app {} over {}".format(appId, appPorts[port]["app"])) + newPort = getNewPort(appPorts, appPorts[port]["app"], appPorts[port]["container"], False) appPorts[newPort] = appPorts[port].copy() appPorts[port] = { "app": appId, @@ -137,7 +135,8 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna disabledApps.append(appId) print("App {} disabled because of port conflict".format(appId)) else: - #print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, appContainer["name"], newPort, port)) + newPort = getNewPort(appPorts, appId, containerName, True) + #print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, containerName, newPort, port)) appPorts[newPort] = { "app": appId, "port": port, @@ -167,7 +166,8 @@ def getPortsV3App(app, appId): else: validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0) elif "requiredPorts" not in appContainer and "requiredUdpPorts" not in appContainer: - validatePort(appContainer["name"], appContainer, getNewPort(appPorts, appId), appId, 0, True) + # if the container does not define a port, assume 3000, and pass it to the container as env var + validatePort(appContainer["name"], appContainer, 3000, appId, 0, True) if "requiredPorts" in appContainer: for port in appContainer["requiredPorts"]: validatePort(appContainer["name"], appContainer, port, appId, 2) diff --git a/app/lib/validate.py b/app/lib/validate.py index db3d5e0..2b6ba09 100644 --- a/app/lib/validate.py +++ b/app/lib/validate.py @@ -11,8 +11,6 @@ import traceback scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") nodeRoot = os.path.join(scriptDir, "..") -with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f: - schemaVersion1 = yaml.safe_load(f) with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f: schemaVersion2 = yaml.safe_load(f) with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f: @@ -24,15 +22,7 @@ with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file: # Validates app data # Returns true if valid, false otherwise def validateApp(app: dict): - if 'version' in app and str(app['version']) == "1": - try: - validate(app, schemaVersion1) - return True - # Catch and log any errors, and return false - except Exception as e: - print(e) - return False - elif 'version' in app and str(app['version']) == "2": + if 'version' in app and str(app['version']) == "2": try: validate(app, schemaVersion2) return True diff --git a/db/dependencies.yml b/db/dependencies.yml index cf5b99d..218cabb 100644 --- a/db/dependencies.yml +++ b/db/dependencies.yml @@ -1,4 +1,4 @@ -compose: v2.10.0 +compose: v2.10.2 dashboard: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e manager: ghcr.io/runcitadel/manager:deno@sha256:38ef8474cc501d3f3e9ea63e73d1c48f848662467ffe5f7f0b9bbb44e04055cf middleware: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae diff --git a/docker-compose.yml b/docker-compose.yml index 6ed0679..6986c56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,6 @@ services: default: ipv4_address: $LND_IP dashboard: - container_name: dashboard image: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e restart: on-failure stop_grace_period: 1m30s @@ -231,6 +230,7 @@ services: networks: default: ipv4_address: $REDIS_IP + networks: default: name: citadel_main_network