mirror of
https://github.com/runcitadel/core.git
synced 2024-09-23 09:40:04 +00:00
Better separation of composegenertor v2 and v3
This commit is contained in:
Aaron Dewes
parent
e381f2a002
commit
33b06241a9
|
|
@ -3,7 +3,7 @@
|
|||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# Main functions
|
||||
from lib.composegenerator.v2.types import App, AppStage3, AppStage2, Container
|
||||
from lib.composegenerator.v2.types import App, AppStage3, AppStage2
|
||||
from lib.composegenerator.shared.const import permissions
|
||||
|
||||
|
||||
|
|
@ -27,28 +27,3 @@ def convertContainersToServices(app: AppStage3) -> AppStage3:
|
|||
del app.containers
|
||||
app.services = services
|
||||
return app
|
||||
|
||||
# Converts the data of every container in app.containers to a volume, which is then added to the app
|
||||
def convertDataDirToVolume(app: App) -> AppStage2:
|
||||
for container in app.containers:
|
||||
# Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
|
||||
# Also, a datadir shouldn't start with a /
|
||||
for dataDir in container.data:
|
||||
if dataDir.find("..") == -1 and dataDir[0] != "/":
|
||||
container.volumes.append(
|
||||
'${APP_DATA_DIR}/' + dataDir)
|
||||
else:
|
||||
print("Data dir " + dataDir +
|
||||
" contains invalid characters")
|
||||
del container.data
|
||||
if container.bitcoin_mount_dir != None:
|
||||
if not 'bitcoind' in container.permissions:
|
||||
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
|
||||
# Skip this container
|
||||
continue
|
||||
# Also skip the container if container.bitcoin_mount_dir contains a :
|
||||
if container.bitcoin_mount_dir.find(":") == -1:
|
||||
container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir)
|
||||
del container.bitcoin_mount_dir
|
||||
|
||||
return app
|
||||
|
|
|
|||
|
|
@ -5,54 +5,21 @@
|
|||
import json
|
||||
from os import path
|
||||
import random
|
||||
from lib.composegenerator.v2.types import ContainerStage2, NetworkConfig
|
||||
from app.lib.composegenerator.v2.utils.networking import getContainerHiddenService
|
||||
from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container
|
||||
from lib.citadelutils import parse_dotenv
|
||||
from dacite import from_dict
|
||||
|
||||
def getFreePort(networkingFile: str, appId: str):
|
||||
# Ports used currently in Citadel
|
||||
usedPorts = [
|
||||
# Dashboard
|
||||
80,
|
||||
# Sometimes used by nginx with some setups
|
||||
433,
|
||||
# Dashboard SSL
|
||||
443,
|
||||
# Bitcoin Core P2P
|
||||
8333,
|
||||
# LND gRPC
|
||||
10009,
|
||||
# LND REST
|
||||
8080,
|
||||
# Electrum Server
|
||||
50001,
|
||||
# Tor Proxy
|
||||
9050,
|
||||
]
|
||||
networkingData = {}
|
||||
if path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
if 'ports' in networkingData:
|
||||
usedPorts += list(networkingData['ports'].values())
|
||||
def getMainContainer(app: App) -> Container:
|
||||
if len(app.containers) == 1:
|
||||
return app.containers[0]
|
||||
else:
|
||||
networkingData['ports'] = {}
|
||||
|
||||
if appId in networkingData['ports']:
|
||||
return networkingData['ports'][appId]
|
||||
|
||||
while True:
|
||||
port = str(random.randint(1024, 49151))
|
||||
if port not in usedPorts:
|
||||
# Check if anyhing is listening on the specific port
|
||||
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
|
||||
networkingData['ports'][appId] = port
|
||||
break
|
||||
|
||||
with open(networkingFile, 'w') as f:
|
||||
json.dump(networkingData, f)
|
||||
|
||||
return port
|
||||
for container in app.containers:
|
||||
# Main is recommended, support web for easier porting from Umbrel
|
||||
if container.name == 'main' or container.name == 'web':
|
||||
return container
|
||||
# Fallback to first container
|
||||
return app.containers[0]
|
||||
|
||||
def assignIpV4(appId: str, containerName: str):
|
||||
scriptDir = path.dirname(path.realpath(__file__))
|
||||
|
|
@ -161,30 +128,44 @@ def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFil
|
|||
json.dump(networkingData, f)
|
||||
return container
|
||||
|
||||
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
|
||||
for container in app.containers:
|
||||
if container.network_mode and container.network_mode == "host":
|
||||
continue
|
||||
if container.noNetwork:
|
||||
# Check if port is defined for the container
|
||||
if container.port:
|
||||
raise Exception("Port defined for container without network")
|
||||
if getMainContainer(app).name == container.name:
|
||||
raise Exception("Main container without network")
|
||||
# Skip this iteration of the loop
|
||||
continue
|
||||
|
||||
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
|
||||
# Strip leading/trailing whitespace from container.name
|
||||
container.name = container.name.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
|
||||
raise Exception("Newline or space in container name")
|
||||
container = assignIp(container, app.metadata.id,
|
||||
networkingFile, envFile)
|
||||
|
||||
env_var = "APP_{}_{}_PORT".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
return app
|
||||
|
||||
port = getFreePort(networkingFile, appId)
|
||||
def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3:
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
hiddenServices = ""
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
mainContainer = getMainContainer(app)
|
||||
|
||||
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
|
||||
with open(envFile, 'a') as f:
|
||||
f.write("{}={}\n".format(env_var, port))
|
||||
|
||||
# This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
|
||||
# where the outer {{ }} will be replaced by {} in the returned string
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
for container in app.containers:
|
||||
if container.network_mode and container.network_mode == "host":
|
||||
continue
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
app.metadata.id.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
hiddenServices += getContainerHiddenService(
|
||||
app.metadata, container, dotEnv[env_var], container.name == mainContainer.name)
|
||||
if container.hiddenServicePorts:
|
||||
del container.hiddenServicePorts
|
||||
|
||||
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
|
||||
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
|
||||
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
|
||||
f.write(hiddenServices)
|
||||
return app
|
||||
|
|
|
|||
|
|
@ -3,15 +3,34 @@
|
|||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp
|
||||
from lib.composegenerator.v2.networking import configureHiddenServices, configureIps, configureMainPort
|
||||
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
|
||||
from lib.composegenerator.v2.networking import configureMainPort
|
||||
from lib.composegenerator.shared.networking import configureHiddenServices, configureIps
|
||||
from lib.composegenerator.shared.main import convertContainerPermissions, convertContainersToServices
|
||||
from lib.composegenerator.shared.env import validateEnv
|
||||
from lib.citadelutils import classToDict
|
||||
import os
|
||||
|
||||
def convertDataDirToVolumeGen2(app: App) -> AppStage2:
|
||||
app = convertDataDirToVolume(app)
|
||||
for container in app.containers:
|
||||
# Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
|
||||
# Also, a datadir shouldn't start with a /
|
||||
for dataDir in container.data:
|
||||
if dataDir.find("..") == -1 and dataDir[0] != "/":
|
||||
container.volumes.append(
|
||||
'${APP_DATA_DIR}/' + dataDir)
|
||||
else:
|
||||
print("Data dir " + dataDir +
|
||||
" contains invalid characters")
|
||||
del container.data
|
||||
if container.bitcoin_mount_dir != None:
|
||||
if not 'bitcoind' in container.permissions:
|
||||
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
|
||||
# Skip this container
|
||||
continue
|
||||
# Also skip the container if container.bitcoin_mount_dir contains a :
|
||||
if container.bitcoin_mount_dir.find(":") == -1:
|
||||
container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir)
|
||||
del container.bitcoin_mount_dir
|
||||
if container.lnd_mount_dir != None:
|
||||
if not 'lnd' in container.permissions:
|
||||
print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name))
|
||||
|
|
|
|||
|
|
@ -2,14 +2,58 @@
|
|||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from app.lib.citadelutils import parse_dotenv
|
||||
from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container
|
||||
from lib.citadelutils import parse_dotenv
|
||||
import json
|
||||
from os import path
|
||||
import os
|
||||
import random
|
||||
from lib.composegenerator.v2.utils.networking import getContainerHiddenService
|
||||
from lib.composegenerator.shared.networking import assignIp, assignPort
|
||||
from lib.composegenerator.shared.networking import assignIp
|
||||
|
||||
def getFreePort(networkingFile: str, appId: str):
|
||||
# Ports used currently in Citadel
|
||||
usedPorts = [
|
||||
# Dashboard
|
||||
80,
|
||||
# Sometimes used by nginx with some setups
|
||||
433,
|
||||
# Dashboard SSL
|
||||
443,
|
||||
# Bitcoin Core P2P
|
||||
8333,
|
||||
# LND gRPC
|
||||
10009,
|
||||
# LND REST
|
||||
8080,
|
||||
# Electrum Server
|
||||
50001,
|
||||
# Tor Proxy
|
||||
9050,
|
||||
]
|
||||
networkingData = {}
|
||||
if path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
if 'ports' in networkingData:
|
||||
usedPorts += list(networkingData['ports'].values())
|
||||
else:
|
||||
networkingData['ports'] = {}
|
||||
|
||||
if appId in networkingData['ports']:
|
||||
return networkingData['ports'][appId]
|
||||
|
||||
while True:
|
||||
port = str(random.randint(1024, 49151))
|
||||
if port not in usedPorts:
|
||||
# Check if anyhing is listening on the specific port
|
||||
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
|
||||
networkingData['ports'][appId] = port
|
||||
break
|
||||
|
||||
with open(networkingFile, 'w') as f:
|
||||
json.dump(networkingData, f)
|
||||
|
||||
return port
|
||||
|
||||
def getMainContainer(app: App) -> Container:
|
||||
if len(app.containers) == 1:
|
||||
|
|
@ -22,6 +66,32 @@ def getMainContainer(app: App) -> Container:
|
|||
# Fallback to first container
|
||||
return app.containers[0]
|
||||
|
||||
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
|
||||
# Strip leading/trailing whitespace from container.name
|
||||
container.name = container.name.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
|
||||
raise Exception("Newline or space in container name")
|
||||
|
||||
env_var = "APP_{}_{}_PORT".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
|
||||
port = getFreePort(networkingFile, appId)
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
|
||||
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
|
||||
with open(envFile, 'a') as f:
|
||||
f.write("{}={}\n".format(env_var, port))
|
||||
|
||||
# This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
|
||||
# where the outer {{ }} will be replaced by {} in the returned string
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
|
||||
|
||||
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
|
||||
registryFile = path.join(nodeRoot, "apps", "registry.json")
|
||||
|
|
@ -83,46 +153,3 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
|
|||
json.dump(registry, f, indent=4, sort_keys=True)
|
||||
|
||||
return app
|
||||
|
||||
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
|
||||
for container in app.containers:
|
||||
if container.network_mode and container.network_mode == "host":
|
||||
continue
|
||||
if container.noNetwork:
|
||||
# Check if port is defined for the container
|
||||
if container.port:
|
||||
raise Exception("Port defined for container without network")
|
||||
if getMainContainer(app).name == container.name:
|
||||
raise Exception("Main container without network")
|
||||
# Skip this iteration of the loop
|
||||
continue
|
||||
|
||||
container = assignIp(container, app.metadata.id,
|
||||
networkingFile, envFile)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3:
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
hiddenServices = ""
|
||||
|
||||
mainContainer = getMainContainer(app)
|
||||
|
||||
for container in app.containers:
|
||||
if container.network_mode and container.network_mode == "host":
|
||||
continue
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
app.metadata.id.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
hiddenServices += getContainerHiddenService(
|
||||
app.metadata, container, dotEnv[env_var], container.name == mainContainer.name)
|
||||
if container.hiddenServicePorts:
|
||||
del container.hiddenServicePorts
|
||||
|
||||
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
|
||||
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
|
||||
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
|
||||
f.write(hiddenServices)
|
||||
return app
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@
|
|||
import os
|
||||
|
||||
from lib.citadelutils import classToDict
|
||||
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainersToServices
|
||||
from lib.composegenerator.shared.main import convertContainersToServices
|
||||
from lib.composegenerator.shared.env import validateEnv
|
||||
from lib.composegenerator.v2.networking import configureIps, configureHiddenServices
|
||||
from lib.composegenerator.shared.networking import configureIps, configureHiddenServices
|
||||
|
||||
from lib.composegenerator.v3.types import App, AppStage2, AppStage4, generateApp
|
||||
from lib.composegenerator.v3.networking import configureMainPort
|
||||
|
|
|
|||
|
|
@ -2,12 +2,10 @@
|
|||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from lib.composegenerator.v3.types import App, AppStage2, AppStage3, Container
|
||||
from lib.citadelutils import parse_dotenv
|
||||
from lib.composegenerator.v3.types import App, AppStage2, AppStage3
|
||||
import json
|
||||
from os import path
|
||||
import random
|
||||
from lib.composegenerator.shared.networking import assignIp, assignPort
|
||||
from lib.composegenerator.shared.networking import assignIp
|
||||
|
||||
def getMainContainerIndex(app: App):
|
||||
if len(app.containers) == 1:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import os
|
|||
import yaml
|
||||
import traceback
|
||||
|
||||
from lib.composegenerator.v2.networking import getMainContainer
|
||||
from lib.composegenerator.shared.networking import assignIpV4
|
||||
from lib.entropy import deriveEntropy
|
||||
from typing import List
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user