diff --git a/app/lib/composegenerator/shared/main.py b/app/lib/composegenerator/shared/main.py index da3c553..bcab1de 100644 --- a/app/lib/composegenerator/shared/main.py +++ b/app/lib/composegenerator/shared/main.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # Main functions -from lib.composegenerator.v2.types import App, AppStage3, AppStage2, Container +from lib.composegenerator.v2.types import App, AppStage3, AppStage2 from lib.composegenerator.shared.const import permissions @@ -27,28 +27,3 @@ def convertContainersToServices(app: AppStage3) -> AppStage3: del app.containers app.services = services return app - -# Converts the data of every container in app.containers to a volume, which is then added to the app -def convertDataDirToVolume(app: App) -> AppStage2: - for container in app.containers: - # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes - # Also, a datadir shouldn't start with a / - for dataDir in container.data: - if dataDir.find("..") == -1 and dataDir[0] != "/": - container.volumes.append( - '${APP_DATA_DIR}/' + dataDir) - else: - print("Data dir " + dataDir + - " contains invalid characters") - del container.data - if container.bitcoin_mount_dir != None: - if not 'bitcoind' in container.permissions: - print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name)) - # Skip this container - continue - # Also skip the container if container.bitcoin_mount_dir contains a : - if container.bitcoin_mount_dir.find(":") == -1: - container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir) - del container.bitcoin_mount_dir - - return app diff --git a/app/lib/composegenerator/shared/networking.py b/app/lib/composegenerator/shared/networking.py index ff9bc98..55422b8 100644 --- a/app/lib/composegenerator/shared/networking.py +++ b/app/lib/composegenerator/shared/networking.py @@ -5,54 +5,21 @@ import json from os import path import random -from lib.composegenerator.v2.types import ContainerStage2, NetworkConfig +from app.lib.composegenerator.v2.utils.networking import getContainerHiddenService +from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container from lib.citadelutils import parse_dotenv from dacite import from_dict -def getFreePort(networkingFile: str, appId: str): - # Ports used currently in Citadel - usedPorts = [ - # Dashboard - 80, - # Sometimes used by nginx with some setups - 433, - # Dashboard SSL - 443, - # Bitcoin Core P2P - 8333, - # LND gRPC - 10009, - # LND REST - 8080, - # Electrum Server - 50001, - # Tor Proxy - 9050, - ] - networkingData = {} - if path.isfile(networkingFile): - with open(networkingFile, 'r') as f: - networkingData = json.load(f) - if 'ports' in networkingData: - usedPorts += list(networkingData['ports'].values()) +def getMainContainer(app: App) -> Container: + if len(app.containers) == 1: + return app.containers[0] else: - networkingData['ports'] = {} - - if appId in networkingData['ports']: - return networkingData['ports'][appId] - - while True: - port = str(random.randint(1024, 49151)) - if port not in usedPorts: - # Check if anyhing is listening on the specific port - if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: - networkingData['ports'][appId] = port - break - - with open(networkingFile, 'w') as f: - json.dump(networkingData, f) - - return port + for container in app.containers: + # Main is recommended, support web for easier porting from Umbrel + if container.name == 'main' or container.name == 'web': + return container + # Fallback to first container + return app.containers[0] def assignIpV4(appId: str, containerName: str): scriptDir = path.dirname(path.realpath(__file__)) @@ -161,30 +128,44 @@ def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFil json.dump(networkingData, f) return container +def configureIps(app: AppStage2, networkingFile: str, envFile: str): + for container in app.containers: + if container.network_mode and container.network_mode == "host": + continue + if container.noNetwork: + # Check if port is defined for the container + if container.port: + raise Exception("Port defined for container without network") + if getMainContainer(app).name == container.name: + raise Exception("Main container without network") + # Skip this iteration of the loop + continue -def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): - # Strip leading/trailing whitespace from container.name - container.name = container.name.strip() - # If the name still contains a newline, throw an error - if container.name.find("\n") != -1 or container.name.find(" ") != -1: - raise Exception("Newline or space in container name") + container = assignIp(container, app.metadata.id, + networkingFile, envFile) - env_var = "APP_{}_{}_PORT".format( - appId.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) + return app - port = getFreePort(networkingFile, appId) +def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3: + dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) + hiddenServices = "" - dotEnv = parse_dotenv(envFile) - if env_var in dotEnv and str(dotEnv[env_var]) == str(port): - return {"port": port, "env_var": "${{{}}}".format(env_var)} + mainContainer = getMainContainer(app) - # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile - with open(envFile, 'a') as f: - f.write("{}={}\n".format(env_var, port)) - - # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} - # where the outer {{ }} will be replaced by {} in the returned string - return {"port": port, "env_var": "${{{}}}".format(env_var)} + for container in app.containers: + if container.network_mode and container.network_mode == "host": + continue + env_var = "APP_{}_{}_IP".format( + app.metadata.id.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + hiddenServices += getContainerHiddenService( + app.metadata, container, dotEnv[env_var], container.name == mainContainer.name) + if container.hiddenServicePorts: + del container.hiddenServicePorts + torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] + torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] + with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: + f.write(hiddenServices) + return app diff --git a/app/lib/composegenerator/v2/generate.py b/app/lib/composegenerator/v2/generate.py index dfcd38d..fbb08bd 100644 --- a/app/lib/composegenerator/v2/generate.py +++ b/app/lib/composegenerator/v2/generate.py @@ -3,15 +3,34 @@ # SPDX-License-Identifier: GPL-3.0-or-later from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp -from lib.composegenerator.v2.networking import configureHiddenServices, configureIps, configureMainPort -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices +from lib.composegenerator.v2.networking import configureMainPort +from lib.composegenerator.shared.networking import configureHiddenServices, configureIps +from lib.composegenerator.shared.main import convertContainerPermissions, convertContainersToServices from lib.composegenerator.shared.env import validateEnv from lib.citadelutils import classToDict import os def convertDataDirToVolumeGen2(app: App) -> AppStage2: - app = convertDataDirToVolume(app) for container in app.containers: + # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes + # Also, a datadir shouldn't start with a / + for dataDir in container.data: + if dataDir.find("..") == -1 and dataDir[0] != "/": + container.volumes.append( + '${APP_DATA_DIR}/' + dataDir) + else: + print("Data dir " + dataDir + + " contains invalid characters") + del container.data + if container.bitcoin_mount_dir != None: + if not 'bitcoind' in container.permissions: + print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name)) + # Skip this container + continue + # Also skip the container if container.bitcoin_mount_dir contains a : + if container.bitcoin_mount_dir.find(":") == -1: + container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir) + del container.bitcoin_mount_dir if container.lnd_mount_dir != None: if not 'lnd' in container.permissions: print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name)) diff --git a/app/lib/composegenerator/v2/networking.py b/app/lib/composegenerator/v2/networking.py index a69540d..eb01041 100644 --- a/app/lib/composegenerator/v2/networking.py +++ b/app/lib/composegenerator/v2/networking.py @@ -2,14 +2,58 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +from app.lib.citadelutils import parse_dotenv from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container -from lib.citadelutils import parse_dotenv import json from os import path +import os import random -from lib.composegenerator.v2.utils.networking import getContainerHiddenService -from lib.composegenerator.shared.networking import assignIp, assignPort +from lib.composegenerator.shared.networking import assignIp +def getFreePort(networkingFile: str, appId: str): + # Ports used currently in Citadel + usedPorts = [ + # Dashboard + 80, + # Sometimes used by nginx with some setups + 433, + # Dashboard SSL + 443, + # Bitcoin Core P2P + 8333, + # LND gRPC + 10009, + # LND REST + 8080, + # Electrum Server + 50001, + # Tor Proxy + 9050, + ] + networkingData = {} + if path.isfile(networkingFile): + with open(networkingFile, 'r') as f: + networkingData = json.load(f) + if 'ports' in networkingData: + usedPorts += list(networkingData['ports'].values()) + else: + networkingData['ports'] = {} + + if appId in networkingData['ports']: + return networkingData['ports'][appId] + + while True: + port = str(random.randint(1024, 49151)) + if port not in usedPorts: + # Check if anyhing is listening on the specific port + if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0: + networkingData['ports'][appId] = port + break + + with open(networkingFile, 'w') as f: + json.dump(networkingData, f) + + return port def getMainContainer(app: App) -> Container: if len(app.containers) == 1: @@ -22,6 +66,32 @@ def getMainContainer(app: App) -> Container: # Fallback to first container return app.containers[0] +def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): + # Strip leading/trailing whitespace from container.name + container.name = container.name.strip() + # If the name still contains a newline, throw an error + if container.name.find("\n") != -1 or container.name.find(" ") != -1: + raise Exception("Newline or space in container name") + + env_var = "APP_{}_{}_PORT".format( + appId.upper().replace("-", "_"), + container.name.upper().replace("-", "_") + ) + + port = getFreePort(networkingFile, appId) + + dotEnv = parse_dotenv(envFile) + if env_var in dotEnv and str(dotEnv[env_var]) == str(port): + return {"port": port, "env_var": "${{{}}}".format(env_var)} + + # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile + with open(envFile, 'a') as f: + f.write("{}={}\n".format(env_var, port)) + + # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }} + # where the outer {{ }} will be replaced by {} in the returned string + return {"port": port, "env_var": "${{{}}}".format(env_var)} + def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: registryFile = path.join(nodeRoot, "apps", "registry.json") @@ -83,46 +153,3 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3: json.dump(registry, f, indent=4, sort_keys=True) return app - -def configureIps(app: AppStage2, networkingFile: str, envFile: str): - for container in app.containers: - if container.network_mode and container.network_mode == "host": - continue - if container.noNetwork: - # Check if port is defined for the container - if container.port: - raise Exception("Port defined for container without network") - if getMainContainer(app).name == container.name: - raise Exception("Main container without network") - # Skip this iteration of the loop - continue - - container = assignIp(container, app.metadata.id, - networkingFile, envFile) - - return app - - -def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3: - dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) - hiddenServices = "" - - mainContainer = getMainContainer(app) - - for container in app.containers: - if container.network_mode and container.network_mode == "host": - continue - env_var = "APP_{}_{}_IP".format( - app.metadata.id.upper().replace("-", "_"), - container.name.upper().replace("-", "_") - ) - hiddenServices += getContainerHiddenService( - app.metadata, container, dotEnv[env_var], container.name == mainContainer.name) - if container.hiddenServicePorts: - del container.hiddenServicePorts - - torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] - torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] - with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: - f.write(hiddenServices) - return app diff --git a/app/lib/composegenerator/v3/generate.py b/app/lib/composegenerator/v3/generate.py index 3e2789f..1164bb1 100644 --- a/app/lib/composegenerator/v3/generate.py +++ b/app/lib/composegenerator/v3/generate.py @@ -5,9 +5,9 @@ import os from lib.citadelutils import classToDict -from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainersToServices +from lib.composegenerator.shared.main import convertContainersToServices from lib.composegenerator.shared.env import validateEnv -from lib.composegenerator.v2.networking import configureIps, configureHiddenServices +from lib.composegenerator.shared.networking import configureIps, configureHiddenServices from lib.composegenerator.v3.types import App, AppStage2, AppStage4, generateApp from lib.composegenerator.v3.networking import configureMainPort diff --git a/app/lib/composegenerator/v3/networking.py b/app/lib/composegenerator/v3/networking.py index 2e25f55..ac784d0 100644 --- a/app/lib/composegenerator/v3/networking.py +++ b/app/lib/composegenerator/v3/networking.py @@ -2,12 +2,10 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from lib.composegenerator.v3.types import App, AppStage2, AppStage3, Container -from lib.citadelutils import parse_dotenv +from lib.composegenerator.v3.types import App, AppStage2, AppStage3 import json from os import path -import random -from lib.composegenerator.shared.networking import assignIp, assignPort +from lib.composegenerator.shared.networking import assignIp def getMainContainerIndex(app: App): if len(app.containers) == 1: diff --git a/app/lib/metadata.py b/app/lib/metadata.py index 69b0f1e..246ab62 100644 --- a/app/lib/metadata.py +++ b/app/lib/metadata.py @@ -6,7 +6,6 @@ import os import yaml import traceback -from lib.composegenerator.v2.networking import getMainContainer from lib.composegenerator.shared.networking import assignIpV4 from lib.entropy import deriveEntropy from typing import List