forked from michael.heier/citadel-core
Refactor the app.yml compiler
This commit is contained in:
Aaron Dewes
parent
7b156dea7b
commit
ca1a3e4d25
|
|
@ -49,7 +49,7 @@ elif args.action == 'download':
|
|||
updateRepos()
|
||||
exit(0)
|
||||
elif args.action == 'update':
|
||||
if(args.invoked_by_configure):
|
||||
if args.invoked_by_configure:
|
||||
update(args.app)
|
||||
else:
|
||||
os.system(os.path.join(nodeRoot, "scripts", "configure"))
|
||||
|
|
@ -64,7 +64,7 @@ elif args.action == 'update':
|
|||
elif args.action == 'update-online':
|
||||
updateRepos()
|
||||
print("Downloaded all updates")
|
||||
if(args.invoked_by_configure):
|
||||
if args.invoked_by_configure:
|
||||
update(args.app)
|
||||
else:
|
||||
os.system(os.path.join(nodeRoot, "scripts", "configure"))
|
||||
|
|
@ -112,7 +112,7 @@ elif args.action == 'stop':
|
|||
print("No app provided")
|
||||
exit(1)
|
||||
userData = getUserData()
|
||||
if(args.app == "installed"):
|
||||
if args.app == "installed":
|
||||
if "installedApps" in userData:
|
||||
stopInstalled()
|
||||
exit(0)
|
||||
|
|
@ -124,7 +124,7 @@ elif args.action == 'start':
|
|||
exit(1)
|
||||
|
||||
userData = getUserData()
|
||||
if(args.app == "installed"):
|
||||
if args.app == "installed":
|
||||
if "installedApps" in userData:
|
||||
startInstalled()
|
||||
exit(0)
|
||||
|
|
|
|||
|
|
@ -186,6 +186,10 @@
|
|||
"type": ["number", "array"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"restarts": {
|
||||
"type": "string",
|
||||
"description": "When the container should restart. Can be 'always' or 'on-failure'."
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
|
||||
import re
|
||||
|
||||
from click import types
|
||||
|
||||
# Helper functions
|
||||
# Return a list of env vars in a string, supports both $NAM§ and ${NAME} format for the env var
|
||||
# This can potentially be used to get around permissions, so this check is critical for security
|
||||
|
|
@ -42,20 +44,59 @@ def parse_dotenv(file_path):
|
|||
exit(1)
|
||||
return envVars
|
||||
|
||||
# Combines two objects
|
||||
# Combines an object and a class
|
||||
# If the key exists in both objects, the value of the second object is used
|
||||
# If the key does not exist in the first object, the value from the second object is used
|
||||
# If a key contains a list, the second object's list is appended to the first object's list
|
||||
# If a key contains another object, these objects are combined
|
||||
def combineObjects(obj1: dict, obj2: dict):
|
||||
for key in obj2:
|
||||
if key in obj1:
|
||||
if isinstance(obj1[key], list):
|
||||
obj1[key] = obj1[key] + obj2[key]
|
||||
elif isinstance(obj1[key], dict):
|
||||
obj1[key] = combineObjects(obj1[key], obj2[key])
|
||||
else:
|
||||
obj1[key] = obj2[key]
|
||||
def combineObjectAndClass(theClass, obj: dict):
|
||||
for key, value in obj.items():
|
||||
if key in theClass.__dict__:
|
||||
if isinstance(value, list):
|
||||
if isinstance(theClass.__dict__[key], list):
|
||||
theClass.__dict__[key].extend(value)
|
||||
else:
|
||||
obj1[key] = obj2[key]
|
||||
return obj1
|
||||
theClass.__dict__[key] = [theClass.__dict__[key]] + value
|
||||
elif isinstance(value, dict):
|
||||
if isinstance(theClass.__dict__[key], dict):
|
||||
theClass.__dict__[key].update(value)
|
||||
else:
|
||||
theClass.__dict__[key] = {theClass.__dict__[key]: value}
|
||||
else:
|
||||
theClass.__dict__[key] = value
|
||||
else:
|
||||
theClass.__dict__[key] = value
|
||||
|
||||
def is_builtin_type(obj):
|
||||
return isinstance(obj, (int, float, str, bool, list, dict, types.ParamType))
|
||||
|
||||
# Convert a class to a dict
|
||||
# Also strip any class member which is null or empty
|
||||
def classToDict(theClass):
|
||||
obj: dict = {}
|
||||
for key, value in theClass.__dict__.items():
|
||||
if value is None or (isinstance(value, list) and len(value) == 0):
|
||||
continue
|
||||
if isinstance(value, list):
|
||||
for element in value:
|
||||
newList = []
|
||||
if is_builtin_type(element):
|
||||
newList.append(element)
|
||||
else:
|
||||
newList.append(classToDict(element))
|
||||
obj[key] = newList
|
||||
elif isinstance(value, dict):
|
||||
newDict = {}
|
||||
for key, value in value.items():
|
||||
if is_builtin_type(value):
|
||||
newDict[key] = value
|
||||
else:
|
||||
newDict[key] = classToDict(value)
|
||||
obj[key] = newDict
|
||||
elif is_builtin_type(value):
|
||||
obj[key] = value
|
||||
else:
|
||||
#print(value)
|
||||
obj[key] = classToDict(value)
|
||||
return obj
|
||||
|
||||
|
|
|
|||
|
|
@ -5,38 +5,39 @@
|
|||
def permissions():
|
||||
return {
|
||||
"lnd": {
|
||||
"environment_allow": {
|
||||
"LND_IP": "${LND_IP}",
|
||||
"LND_GRPC_PORT": "${LND_GRPC_PORT}",
|
||||
"LND_REST_PORT": "${LND_REST_PORT}",
|
||||
"BITCOIN_NETWORK": "${BITCOIN_NETWORK}"
|
||||
},
|
||||
"environment_allow": [
|
||||
"LND_IP",
|
||||
"LND_GRPC_PORT",
|
||||
"LND_REST_PORT",
|
||||
"BITCOIN_NETWORK"
|
||||
],
|
||||
"volumes": [
|
||||
'${LND_DATA_DIR}:/lnd:ro'
|
||||
]
|
||||
},
|
||||
"bitcoind": {
|
||||
"environment_allow": {
|
||||
"BITCOIN_IP": "${BITCOIN_IP}",
|
||||
"BITCOIN_NETWORK": "${BITCOIN_NETWORK}",
|
||||
"BITCOIN_P2P_PORT": "${BITCOIN_P2P_PORT}",
|
||||
"BITCOIN_RPC_PORT": "${BITCOIN_RPC_PORT}",
|
||||
"BITCOIN_RPC_USER": "${BITCOIN_RPC_USER}",
|
||||
"BITCOIN_RPC_PASS": "${BITCOIN_RPC_PASS}",
|
||||
"BITCOIN_RPC_AUTH": "${BITCOIN_RPC_AUTH}",
|
||||
"BITCOIN_ZMQ_RAWBLOCK_PORT": "${BITCOIN_ZMQ_RAWBLOCK_PORT}",
|
||||
"BITCOIN_ZMQ_RAWTX_PORT": "${BITCOIN_ZMQ_RAWTX_PORT}",
|
||||
"BITCOIN_ZMQ_HASHBLOCK_PORT": "${BITCOIN_ZMQ_HASHBLOCK_PORT}",
|
||||
},
|
||||
"environment_allow": [
|
||||
"BITCOIN_IP",
|
||||
"BITCOIN_NETWORK",
|
||||
"BITCOIN_P2P_PORT",
|
||||
"BITCOIN_RPC_PORT",
|
||||
"BITCOIN_RPC_USER",
|
||||
"BITCOIN_RPC_PASS",
|
||||
"BITCOIN_RPC_AUTH",
|
||||
"BITCOIN_ZMQ_RAWBLOCK_PORT",
|
||||
"BITCOIN_ZMQ_RAWTX_PORT",
|
||||
"BITCOIN_ZMQ_HASHBLOCK_PORT",
|
||||
],
|
||||
"volumes": [
|
||||
"${BITCOIN_DATA_DIR}:/bitcoin:ro"
|
||||
]
|
||||
},
|
||||
"electrum": {
|
||||
"environment_allow": {
|
||||
"ELECTRUM_IP": "${ELECTRUM_IP}",
|
||||
"ELECTRUM_PORT": "${ELECTRUM_PORT}",
|
||||
}
|
||||
"environment_allow": [
|
||||
"ELECTRUM_IP",
|
||||
"ELECTRUM_PORT",
|
||||
],
|
||||
"volumes": []
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import re
|
||||
from lib.composegenerator.v1.types import App
|
||||
from lib.composegenerator.shared.const import always_allowed_env
|
||||
from lib.citadelutils import checkArrayContainsAllElements, getEnvVars
|
||||
|
||||
|
|
@ -10,7 +11,7 @@ def validateEnvByValue(env: list, allowed: list, app_name: str):
|
|||
# Combine always_allowed_env with allowed into one list
|
||||
# Then check if all elements in env are in the resulting list
|
||||
all_allowed = allowed + always_allowed_env
|
||||
if(not checkArrayContainsAllElements(env, all_allowed)):
|
||||
if not checkArrayContainsAllElements(env, all_allowed):
|
||||
# This has a weird syntax, and it confuses VSCode, but it works
|
||||
validation_regex = r"APP_{}(\S+)".format(
|
||||
app_name.upper().replace("-", "_"))
|
||||
|
|
@ -23,24 +24,24 @@ def validateEnvByValue(env: list, allowed: list, app_name: str):
|
|||
return True
|
||||
|
||||
|
||||
def validateEnv(app: dict):
|
||||
def validateEnv(app: App):
|
||||
# For every container of the app, check if all env vars in the strings in environment are defined in env
|
||||
for container in app['containers']:
|
||||
if 'environment' in container:
|
||||
if 'environment_allow' in container:
|
||||
existingEnv = list(container['environment_allow'].keys())
|
||||
del container['environment_allow']
|
||||
for container in app.containers:
|
||||
if container.environment:
|
||||
if container.environment_allow:
|
||||
existingEnv = container.environment_allow
|
||||
del container.environment_allow
|
||||
else:
|
||||
existingEnv = []
|
||||
# The next step depends on the type of the environment object, which is either a list or dict
|
||||
# If it's a list, split every string in it by the first=, then run getEnvVars(envVarValue) on it
|
||||
# ON a dict, run getEnvVars(envVarValue) on every value of the environment object
|
||||
# Then check if all env vars returned by getEnvVars are defined in env
|
||||
if(isinstance(container['environment'], list)):
|
||||
if isinstance(container.environment, list):
|
||||
raise Exception("List env vars are no longer supported for container {} of app {}".format(
|
||||
container['name'], app['metadata']['name']))
|
||||
elif(isinstance(container['environment'], dict)):
|
||||
for envVar in container['environment'].values():
|
||||
if(not validateEnvByValue(getEnvVars(envVar), existingEnv, app['metadata']['id'])):
|
||||
container.name, app.metadata.name))
|
||||
elif isinstance(container.environment, dict):
|
||||
for envVar in container.environment.values():
|
||||
if not validateEnvByValue(getEnvVars(envVar), existingEnv, app.metadata.id):
|
||||
raise Exception("Env vars not defined for container {} of app {}".format(
|
||||
container['name'], app['metadata']['name']))
|
||||
container.name, app.metadata.name))
|
||||
|
|
|
|||
|
|
@ -3,64 +3,51 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
# Main functions
|
||||
from lib.citadelutils import combineObjects
|
||||
from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container
|
||||
from lib.composegenerator.shared.const import permissions
|
||||
|
||||
|
||||
def convertContainerPermissions(app):
|
||||
for container in app['containers']:
|
||||
if 'permissions' in container:
|
||||
for permission in container['permissions']:
|
||||
if(permission in permissions()):
|
||||
container = combineObjects(
|
||||
container, permissions()[permission])
|
||||
else:
|
||||
print("Warning: container {} of app {} defines unknown permission {}".format(container['name'], app['metadata']['name'], permission))
|
||||
def convertContainerPermissions(app: App) -> App:
|
||||
for container in app.containers:
|
||||
for permission in container.permissions:
|
||||
if permission in permissions():
|
||||
container.environment_allow.extend(permissions()[permission]['environment_allow'])
|
||||
container.volumes.extend(permissions()[permission]['volumes'])
|
||||
else:
|
||||
print("Warning: container {} of app {} defines unknown permission {}".format(container.name, app.metadata.name, permission))
|
||||
return app
|
||||
|
||||
def convertContainersToServices(app: dict):
|
||||
app['services'] = {}
|
||||
for container in app['containers']:
|
||||
if 'permissions' in container:
|
||||
del container['permissions']
|
||||
app['services'][container['name']] = container
|
||||
del app['services'][container['name']]['name']
|
||||
del app['containers']
|
||||
def convertContainersToServices(app: AppStage3) -> AppStage3:
|
||||
app.services = {}
|
||||
for container in app.containers:
|
||||
if container.permissions:
|
||||
del container.permissions
|
||||
app.services[container.name] = container
|
||||
del app.services[container.name].name
|
||||
del app.containers
|
||||
return app
|
||||
|
||||
# Converts the data of every container in app['containers'] to a volume, which is then added to the app
|
||||
def convertDataDirToVolume(app: dict):
|
||||
for container in app['containers']:
|
||||
# Loop through data dirs in container['data'], if they don't contain a .., add them to container['volumes']
|
||||
# Converts the data of every container in app.containers to a volume, which is then added to the app
|
||||
def convertDataDirToVolume(app: App) -> AppStage2:
|
||||
for container in app.containers:
|
||||
# Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
|
||||
# Also, a datadir shouldn't start with a /
|
||||
if 'data' in container:
|
||||
for dataDir in container['data']:
|
||||
if not 'volumes' in container:
|
||||
container['volumes'] = []
|
||||
if(dataDir.find("..") == -1 and dataDir[0] != "/"):
|
||||
container['volumes'].append(
|
||||
'${APP_DATA_DIR}/' + dataDir)
|
||||
else:
|
||||
print("Data dir " + dataDir +
|
||||
" contains invalid characters")
|
||||
del container['data']
|
||||
if 'bitcoin_mount_dir' in container:
|
||||
if not 'permissions' in container or not 'bitcoind' in container['permissions']:
|
||||
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container['name'], app['metadata']['name']))
|
||||
for dataDir in container.data:
|
||||
if dataDir.find("..") == -1 and dataDir[0] != "/":
|
||||
container.volumes.append(
|
||||
'${APP_DATA_DIR}/' + dataDir)
|
||||
else:
|
||||
print("Data dir " + dataDir +
|
||||
" contains invalid characters")
|
||||
del container.data
|
||||
if container.bitcoin_mount_dir != None:
|
||||
if not 'bitcoind' in container.permissions:
|
||||
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
|
||||
# Skip this container
|
||||
continue
|
||||
if not 'volumes' in container:
|
||||
container['volumes'] = []
|
||||
# Also skip the container if container['bitcoin_mount_dir'] contains a :
|
||||
if(container['bitcoin_mount_dir'].find(":") == -1):
|
||||
container['volumes'].append('${BITCOIN_DATA_DIR}:' + container['bitcoin_mount_dir'] + ':ro')
|
||||
del container['bitcoin_mount_dir']
|
||||
# Also skip the container if container.bitcoin_mount_dir contains a :
|
||||
if container.bitcoin_mount_dir.find(":") == -1:
|
||||
container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir + ':ro')
|
||||
del container.bitcoin_mount_dir
|
||||
|
||||
return app
|
||||
|
||||
def addStopConfig(app: dict):
|
||||
for container in app['containers']:
|
||||
if not 'stop_grace_period' in container:
|
||||
container['stop_grace_period'] = '1m'
|
||||
container['restart'] = "on-failure"
|
||||
return app
|
||||
|
|
|
|||
|
|
@ -2,30 +2,33 @@
|
|||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
from lib.composegenerator.v1.types import App, AppStage4, generateApp
|
||||
from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort
|
||||
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, addStopConfig, convertContainersToServices
|
||||
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
|
||||
from lib.composegenerator.shared.env import validateEnv
|
||||
from lib.citadelutils import classToDict
|
||||
import os
|
||||
|
||||
def createComposeConfigFromV1(app: dict, nodeRoot: str):
|
||||
if("version" in app):
|
||||
if(str(app['version']) != "1"):
|
||||
if "version" in app:
|
||||
if str(app['version']) != "1":
|
||||
print("Warning: app version is not supported")
|
||||
return False
|
||||
envFile = os.path.join(nodeRoot, ".env")
|
||||
networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
|
||||
|
||||
app = convertContainerPermissions(app)
|
||||
validateEnv(app)
|
||||
app = convertDataDirToVolume(app)
|
||||
app = configureIps(app, networkingFile, envFile)
|
||||
app = configureMainPort(app, nodeRoot)
|
||||
app = configureHiddenServices(app, nodeRoot)
|
||||
app = addStopConfig(app)
|
||||
app = convertContainersToServices(app)
|
||||
del app['metadata']
|
||||
if("version" in app):
|
||||
del app["version"]
|
||||
newApp: App = generateApp(app)
|
||||
newApp = convertContainerPermissions(newApp)
|
||||
validateEnv(newApp)
|
||||
newApp = convertDataDirToVolume(newApp)
|
||||
newApp = configureIps(newApp, networkingFile, envFile)
|
||||
newApp = configureMainPort(newApp, nodeRoot)
|
||||
configureHiddenServices(newApp, nodeRoot)
|
||||
finalConfig: AppStage4 = convertContainersToServices(newApp)
|
||||
newApp = classToDict(finalConfig)
|
||||
del newApp['metadata']
|
||||
if "version" in newApp:
|
||||
del newApp["version"]
|
||||
# Set version to 3.8 (current compose file version)
|
||||
app = {'version': '3.8', **app}
|
||||
return app
|
||||
newApp = {'version': '3.8', **newApp}
|
||||
return newApp
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
from dacite import from_dict
|
||||
from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig
|
||||
from lib.citadelutils import parse_dotenv
|
||||
import json
|
||||
from os import path
|
||||
|
|
@ -9,49 +11,51 @@ import random
|
|||
from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService
|
||||
|
||||
|
||||
def assignIp(container: dict, appId: str, networkingFile: str, envFile: str):
|
||||
# Strip leading/trailing whitespace from container['name']
|
||||
container['name'] = container['name'].strip()
|
||||
def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
|
||||
# Strip leading/trailing whitespace from container.name
|
||||
container.name = container.name.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if(container['name'].find("\n") != -1):
|
||||
if container.name.find("\n") != -1:
|
||||
raise Exception("Newline in container name")
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
container['name'].upper().replace("-", "_")
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
|
||||
# can be used
|
||||
usedIps = []
|
||||
networkingData = {}
|
||||
if(path.isfile(networkingFile)):
|
||||
if path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
|
||||
if('ip_addresses' in networkingData):
|
||||
if 'ip_addresses' in networkingData:
|
||||
usedIps = list(networkingData['ip_addresses'].values())
|
||||
else:
|
||||
networkingData['ip_addresses'] = {}
|
||||
networkingData.ip_addresses = {}
|
||||
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
|
||||
# If the IP is already in use, it will be tried again until it's not in use
|
||||
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
|
||||
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
|
||||
# Throw an error, because no more IPs can be used
|
||||
if(len(usedIps) == 235):
|
||||
if len(usedIps) == 235:
|
||||
raise Exception("No more IPs can be used")
|
||||
|
||||
if("{}-{}".format(appId, container['name']) in networkingData['ip_addresses']):
|
||||
ip = networkingData['ip_addresses']["{}-{}".format(appId, container['name'])]
|
||||
if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
|
||||
ip = networkingData['ip_addresses']["{}-{}".format(
|
||||
appId, container.name)]
|
||||
else:
|
||||
while True:
|
||||
ip = "10.21.21." + str(random.randint(20, 255))
|
||||
if(ip not in usedIps):
|
||||
networkingData['ip_addresses']["{}-{}".format(appId, container['name'])] = ip
|
||||
if ip not in usedIps:
|
||||
networkingData['ip_addresses']["{}-{}".format(
|
||||
appId, container.name)] = ip
|
||||
break
|
||||
container['networks'] = {'default': {
|
||||
'ipv4_address': "$" + env_var}}
|
||||
container.networks = from_dict(data_class=NetworkConfig, data={'default': {
|
||||
'ipv4_address': "$" + env_var}})
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if(env_var in dotEnv and str(dotEnv[env_var]) == str(ip)):
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
|
||||
return container
|
||||
|
||||
# Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile
|
||||
|
|
@ -63,21 +67,21 @@ def assignIp(container: dict, appId: str, networkingFile: str, envFile: str):
|
|||
|
||||
|
||||
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
|
||||
# Strip leading/trailing whitespace from container['name']
|
||||
container['name'] = container['name'].strip()
|
||||
# Strip leading/trailing whitespace from container.name
|
||||
container.name = container.name.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if(container['name'].find("\n") != -1 or container['name'].find(" ") != -1):
|
||||
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
|
||||
raise Exception("Newline or space in container name")
|
||||
|
||||
env_var = "APP_{}_{}_PORT".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
container['name'].upper().replace("-", "_")
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
|
||||
port = getFreePort(networkingFile, appId)
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if(env_var in dotEnv and str(dotEnv[env_var]) == str(port)):
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
|
||||
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
|
||||
|
|
@ -88,32 +92,34 @@ def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
|
|||
# where the outer {{ }} will be replaced by {} in the returned string
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
|
||||
def getMainContainer(app: dict):
|
||||
if len(app['containers']) == 1:
|
||||
return app['containers'][0]
|
||||
else:
|
||||
if not 'mainContainer' in app['metadata']:
|
||||
app['metadata']['mainContainer'] = 'main'
|
||||
for container in app['containers']:
|
||||
if container['name'] == app['metadata']['mainContainer']:
|
||||
return container
|
||||
raise Exception("No main container found for app {}".format(app['metadata']['name']))
|
||||
|
||||
def configureMainPort(app: dict, nodeRoot: str):
|
||||
def getMainContainer(app: dict):
|
||||
if len(app.containers) == 1:
|
||||
return app.containers[0]
|
||||
else:
|
||||
if not app.metadata.mainContainer:
|
||||
app.metadata.mainContainer = 'main'
|
||||
for container in app.containers:
|
||||
if container.name == app.metadata.mainContainer:
|
||||
return container
|
||||
raise Exception(
|
||||
"No main container found for app {}".format(app.metadata.name))
|
||||
|
||||
|
||||
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
|
||||
registryFile = path.join(nodeRoot, "apps", "registry.json")
|
||||
registry: list = []
|
||||
if(path.isfile(registryFile)):
|
||||
if path.isfile(registryFile):
|
||||
with open(registryFile, 'r') as f:
|
||||
registry = json.load(f)
|
||||
else:
|
||||
raise Exception("Registry file not found")
|
||||
|
||||
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
|
||||
mainContainer = getMainContainer(app)
|
||||
|
||||
portDetails = assignPort(mainContainer, app['metadata']['id'], path.join(
|
||||
portDetails = assignPort(mainContainer, app.metadata.id, path.join(
|
||||
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
|
||||
containerPort = portDetails['port']
|
||||
portAsEnvVar = portDetails['env_var']
|
||||
|
|
@ -121,37 +127,37 @@ def configureMainPort(app: dict, nodeRoot: str):
|
|||
|
||||
mainPort = False
|
||||
|
||||
if "port" in mainContainer:
|
||||
portToAppend = "{}:{}".format(portAsEnvVar, mainContainer['port'])
|
||||
mainPort = mainContainer['port']
|
||||
del mainContainer['port']
|
||||
if mainContainer.port:
|
||||
portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port)
|
||||
mainPort = mainContainer.port
|
||||
del mainContainer.port
|
||||
else:
|
||||
portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar)
|
||||
|
||||
if "ports" in mainContainer:
|
||||
mainContainer['ports'].append(portToAppend)
|
||||
if mainContainer.ports:
|
||||
mainContainer.ports.append(portToAppend)
|
||||
# Set the main port to the first port in the list, if it contains a :, it's the port after the :
|
||||
# If it doesn't contain a :, it's the port itself
|
||||
if mainPort == False:
|
||||
mainPort = mainContainer['ports'][0]
|
||||
if(mainPort.find(":") != -1):
|
||||
mainPort = mainContainer.ports[0]
|
||||
if mainPort.find(":") != -1:
|
||||
mainPort = mainPort.split(":")[1]
|
||||
else:
|
||||
mainContainer['ports'] = [portToAppend]
|
||||
mainContainer.ports = [portToAppend]
|
||||
if mainPort == False:
|
||||
mainPort = portDetails['port']
|
||||
|
||||
mainContainer = assignIp(mainContainer, app['metadata']['id'], path.join(
|
||||
mainContainer = assignIp(mainContainer, app.metadata.id, path.join(
|
||||
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
|
||||
|
||||
# If the IP wasn't in dotenv before, now it should be
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
|
||||
containerIP = dotEnv['APP_{}_{}_IP'.format(app['metadata']['id'].upper().replace(
|
||||
"-", "_"), mainContainer['name'].upper().replace("-", "_"))]
|
||||
containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace(
|
||||
"-", "_"), mainContainer.name.upper().replace("-", "_"))]
|
||||
|
||||
hiddenservice = getHiddenService(
|
||||
app['metadata']['name'], app['metadata']['id'], containerIP, mainPort)
|
||||
app.metadata.name, app.metadata.id, containerIP, mainPort)
|
||||
|
||||
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
|
||||
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
|
||||
|
|
@ -159,10 +165,10 @@ def configureMainPort(app: dict, nodeRoot: str):
|
|||
f.write(hiddenservice)
|
||||
|
||||
# Also set the port in metadata
|
||||
app['metadata']['port'] = int(containerPort)
|
||||
app.metadata.port = int(containerPort)
|
||||
|
||||
for registryApp in registry:
|
||||
if(registryApp['id'] == app['metadata']['id']):
|
||||
if registryApp['id'] == app.metadata.id:
|
||||
registry[registry.index(registryApp)]['port'] = int(containerPort)
|
||||
break
|
||||
|
||||
|
|
@ -172,48 +178,49 @@ def configureMainPort(app: dict, nodeRoot: str):
|
|||
return app
|
||||
|
||||
|
||||
def configureIps(app: dict, networkingFile: str, envFile: str):
|
||||
for container in app['containers']:
|
||||
if('noNetwork' in container and container['noNetwork']):
|
||||
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
|
||||
for container in app.containers:
|
||||
if container.noNetwork:
|
||||
# Check if port is defined for the container
|
||||
if('port' in container):
|
||||
if container.port:
|
||||
raise Exception("Port defined for container without network")
|
||||
if(app['metadata']['mainContainer'] == container['name']):
|
||||
if app.metadata.mainContainer == container.name:
|
||||
raise Exception("Main container without network")
|
||||
# Skip this iteration of the loop
|
||||
continue
|
||||
|
||||
container = assignIp(container, app['metadata']['id'], networkingFile, envFile)
|
||||
|
||||
container = assignIp(container, app.metadata.id,
|
||||
networkingFile, envFile)
|
||||
|
||||
return app
|
||||
|
||||
def configureHiddenServices(app: dict, nodeRoot: str):
|
||||
|
||||
def configureHiddenServices(app: dict, nodeRoot: str) -> None:
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
hiddenServices = ""
|
||||
|
||||
if len(app['containers']) == 1:
|
||||
mainContainer = app['containers'][0]
|
||||
if len(app.containers) == 1:
|
||||
mainContainer = app.containers[0]
|
||||
else:
|
||||
mainContainer = None
|
||||
if not 'mainContainer' in app['metadata']:
|
||||
app['metadata']['mainContainer'] = 'main'
|
||||
for container in app['containers']:
|
||||
if container['name'] == app['metadata']['mainContainer']:
|
||||
if app.metadata.mainContainer == None:
|
||||
app.metadata.mainContainer = 'main'
|
||||
for container in app.containers:
|
||||
if container.name == app.metadata.mainContainer:
|
||||
mainContainer = container
|
||||
break
|
||||
if mainContainer is None:
|
||||
raise Exception("No main container found")
|
||||
|
||||
for container in app['containers']:
|
||||
|
||||
for container in app.containers:
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
app["metadata"]["id"].upper().replace("-", "_"),
|
||||
container['name'].upper().replace("-", "_")
|
||||
app.metadata.id.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
hiddenServices += getContainerHiddenService(app["metadata"]["name"], app["metadata"]["id"], container, dotEnv[env_var], container["name"] == mainContainer["name"])
|
||||
hiddenServices += getContainerHiddenService(
|
||||
app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name)
|
||||
|
||||
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
|
||||
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
|
||||
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
|
||||
f.write(hiddenServices)
|
||||
|
||||
return app
|
||||
151
app/lib/composegenerator/v1/types.py
Normal file
151
app/lib/composegenerator/v1/types.py
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
from typing import Union
|
||||
from dataclasses import dataclass, field
|
||||
from dacite import from_dict
|
||||
|
||||
@dataclass
|
||||
class Metadata:
|
||||
id: str
|
||||
name: str
|
||||
version: str
|
||||
category: str
|
||||
tagline: str
|
||||
description: str
|
||||
developer: str
|
||||
website: str
|
||||
repo: str
|
||||
support: str
|
||||
gallery: list[str] = field(default_factory=list)
|
||||
dependencies: list[str] = field(default_factory=list)
|
||||
mainContainer: Union[str, None] = None
|
||||
updateContainer: Union[str, None] = None
|
||||
path: str = ""
|
||||
defaultPassword: str = ""
|
||||
torOnly: bool = False
|
||||
|
||||
@dataclass
|
||||
class Container:
|
||||
name: str
|
||||
image: str
|
||||
permissions: list = field(default_factory=list)
|
||||
ports: list = field(default_factory=list)
|
||||
port: Union[int, None] = None
|
||||
environment: Union[dict, None] = None
|
||||
data: list = field(default_factory=list)
|
||||
user: Union[str, None] = None
|
||||
stop_grace_period: str = '1m'
|
||||
depends_on: list = field(default_factory=list)
|
||||
entrypoint: Union[list[str], str] = field(default_factory=list)
|
||||
bitcoin_mount_dir: Union[str, None] = None
|
||||
command: Union[list[str], str] = field(default_factory=list)
|
||||
init: Union[bool, None] = None
|
||||
stop_signal: Union[str, None] = None
|
||||
noNetwork: Union[bool, None] = None
|
||||
needsHiddenService: Union[bool, None] = None
|
||||
hiddenServicePort: Union[int, None] = None
|
||||
hiddenServicePorts: Union[dict, None] = None
|
||||
environment_allow: list = field(default_factory=list)
|
||||
# Only added later
|
||||
volumes: list = field(default_factory=list)
|
||||
restarts: Union[str, None] = None
|
||||
|
||||
@dataclass
|
||||
class App:
|
||||
version: Union[str, int]
|
||||
metadata: Metadata
|
||||
containers: list[Container]
|
||||
|
||||
# Generate an app instance from an app dict
|
||||
def generateApp(appDict):
|
||||
return from_dict(data_class=App, data=appDict)
|
||||
|
||||
@dataclass
|
||||
class Network:
|
||||
ipv4_address: Union[str, None] = None
|
||||
|
||||
@dataclass
|
||||
class NetworkConfig:
|
||||
default: Network
|
||||
|
||||
# After converting data dir and defining volumes, stage 2
|
||||
@dataclass
|
||||
class ContainerStage2:
|
||||
id: str
|
||||
name: str
|
||||
image: str
|
||||
permissions: list[str] = field(default_factory=list)
|
||||
ports: list = field(default_factory=list)
|
||||
environment: Union[dict, None] = None
|
||||
user: Union[str, None] = None
|
||||
stop_grace_period: str = '1m'
|
||||
depends_on: list[str] = field(default_factory=list)
|
||||
entrypoint: Union[list[str], str] = field(default_factory=list)
|
||||
command: Union[list[str], str] = field(default_factory=list)
|
||||
init: Union[bool, None] = None
|
||||
stop_signal: Union[str, None] = None
|
||||
noNetwork: Union[bool, None] = None
|
||||
needsHiddenService: Union[bool, None] = None
|
||||
hiddenServicePort: Union[int, None] = None
|
||||
hiddenServicePorts: Union[dict, None] = None
|
||||
volumes: list[str] = field(default_factory=list)
|
||||
networks: NetworkConfig = field(default_factory=NetworkConfig)
|
||||
restarts: Union[str, None] = None
|
||||
|
||||
@dataclass
|
||||
class AppStage2:
|
||||
version: Union[str, int]
|
||||
metadata: Metadata
|
||||
containers: list[ContainerStage2]
|
||||
|
||||
@dataclass
|
||||
class MetadataStage3:
|
||||
id: str
|
||||
name: str
|
||||
version: str
|
||||
category: str
|
||||
tagline: str
|
||||
description: str
|
||||
developer: str
|
||||
website: str
|
||||
dependencies: list[str]
|
||||
repo: str
|
||||
support: str
|
||||
gallery: list[str]
|
||||
mainContainer: Union[str, None] = None
|
||||
updateContainer: Union[str, None] = None
|
||||
path: str = ""
|
||||
defaultPassword: str = ""
|
||||
torOnly: bool = False
|
||||
|
||||
@dataclass
|
||||
class AppStage3:
|
||||
version: Union[str, int]
|
||||
metadata: MetadataStage3
|
||||
containers: list[ContainerStage2]
|
||||
|
||||
@dataclass
|
||||
class ContainerStage4:
|
||||
id: str
|
||||
name: str
|
||||
image: str
|
||||
ports: list = field(default_factory=list)
|
||||
environment: Union[dict, None] = None
|
||||
user: Union[str, None] = None
|
||||
stop_grace_period: str = '1m'
|
||||
depends_on: list[str] = field(default_factory=list)
|
||||
entrypoint: Union[list[str], str] = field(default_factory=list)
|
||||
command: Union[list[str], str] = field(default_factory=list)
|
||||
init: Union[bool, None] = None
|
||||
stop_signal: Union[str, None] = None
|
||||
noNetwork: Union[bool, None] = None
|
||||
needsHiddenService: Union[bool, None] = None
|
||||
hiddenServicePort: Union[int, None] = None
|
||||
hiddenServicePorts: Union[dict, None] = None
|
||||
volumes: list[str] = field(default_factory=list)
|
||||
networks: NetworkConfig = field(default_factory=NetworkConfig)
|
||||
restarts: Union[str, None] = None
|
||||
|
||||
@dataclass
|
||||
class AppStage4:
|
||||
version: Union[str, int]
|
||||
metadata: MetadataStage3
|
||||
services: list[ContainerStage4]
|
||||
|
|
@ -6,6 +6,8 @@ import json
|
|||
import os
|
||||
import random
|
||||
|
||||
from lib.composegenerator.v1.types import Container
|
||||
|
||||
|
||||
def getFreePort(networkingFile: str, appId: str):
|
||||
# Ports used currently in Citadel
|
||||
|
|
@ -13,22 +15,22 @@ def getFreePort(networkingFile: str, appId: str):
|
|||
usedPorts = [80, 8333, 8332, 28332, 28333, 28334, 10009, 8080, 50001, 9050, 3002, 3000, 3300, 3001, 3004, 25441,
|
||||
3003, 3007, 3006, 3009, 3005, 8898, 3008, 8081, 8082, 8083, 8085, 2222, 8086, 8087, 8008, 8088, 8089, 8091]
|
||||
networkingData = {}
|
||||
if(os.path.isfile(networkingFile)):
|
||||
if os.path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
if('ports' in networkingData):
|
||||
if 'ports' in networkingData:
|
||||
usedPorts += list(networkingData['ports'].values())
|
||||
else:
|
||||
networkingData['ports'] = {}
|
||||
|
||||
if(appId in networkingData['ports']):
|
||||
if appId in networkingData['ports']:
|
||||
return networkingData['ports'][appId]
|
||||
|
||||
while True:
|
||||
port = str(random.randint(1024, 49151))
|
||||
if(port not in usedPorts):
|
||||
if port not in usedPorts:
|
||||
# Check if anyhing is listening on the specific port
|
||||
if(os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0):
|
||||
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
|
||||
networkingData['ports'][appId] = port
|
||||
break
|
||||
|
||||
|
|
@ -63,21 +65,21 @@ def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str:
|
|||
return getHiddenServiceString(appName, appId, appPort, appIp, "80")
|
||||
|
||||
|
||||
def getContainerHiddenService(appName: str, appId: str, container: dict, containerIp: str, isMainContainer: bool) -> str:
|
||||
if not "needsHiddenService" in container and not isMainContainer:
|
||||
def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str:
|
||||
if not container.needsHiddenService and not isMainContainer:
|
||||
return ""
|
||||
if ("ports" in container or not "port" in container) and not "hiddenServicePort" in container and not isMainContainer:
|
||||
if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer:
|
||||
print("Container {} for app {} isn't compatible with hidden service assignment".format(
|
||||
container["name"], appName))
|
||||
container.name, appName))
|
||||
return ""
|
||||
|
||||
if isMainContainer:
|
||||
if not "hiddenServicePorts" in container:
|
||||
if not container.hiddenServicePorts:
|
||||
return ""
|
||||
# hiddenServicePorts is a map of hidden service name to port
|
||||
# We need to generate a hidden service for each one
|
||||
hiddenServices = ""
|
||||
for name, port in container["hiddenServicePorts"].items():
|
||||
for name, port in container.hiddenServicePorts.items():
|
||||
if ".." in name:
|
||||
print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName))
|
||||
|
||||
|
|
@ -88,15 +90,15 @@ def getContainerHiddenService(appName: str, appId: str, container: dict, contain
|
|||
else:
|
||||
hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format(
|
||||
appId, name), port, containerIp, port)
|
||||
del container["hiddenServicePorts"]
|
||||
del container.hiddenServicePorts
|
||||
return hiddenServices
|
||||
|
||||
del container["needsHiddenService"]
|
||||
if not "port" in container:
|
||||
data = getHiddenServiceString(appName + container["name"], "{}-{}".format(
|
||||
appId, container["name"]), container["hiddenServicePort"], containerIp, "80")
|
||||
del container["hiddenServicePort"]
|
||||
del container.needsHiddenService
|
||||
if not container.port:
|
||||
data = getHiddenServiceString(appName + container.name, "{}-{}".format(
|
||||
appId, container.name), container.hiddenServicePort, containerIp, "80")
|
||||
del container.hiddenServicePort
|
||||
return data
|
||||
else:
|
||||
return getHiddenServiceString(appName + container["name"], "{}-{}".format(
|
||||
appId, container["name"]), container["port"], containerIp, container["port"])
|
||||
return getHiddenServiceString(appName + container.name, "{}-{}".format(
|
||||
appId, container.name), container.port, containerIp, container.port)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ def deriveEntropy(identifier: str):
|
|||
seedFile = os.path.join(nodeRoot, "db", "citadel-seed", "seed")
|
||||
alternativeSeedFile = os.path.join(nodeRoot, "db", "citadel-seed", "seed")
|
||||
if not os.path.isfile(seedFile):
|
||||
if(os.path.isfile(alternativeSeedFile)):
|
||||
if os.path.isfile(alternativeSeedFile):
|
||||
seedFile = alternativeSeedFile
|
||||
else:
|
||||
print("No seed file found, exiting...")
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ def update(verbose: bool = False):
|
|||
appYml = os.path.join(appsDir, app, "app.yml")
|
||||
with open(composeFile, "w") as f:
|
||||
appCompose = getApp(appYml, app)
|
||||
if(appCompose):
|
||||
if appCompose:
|
||||
f.write(yaml.dump(appCompose, sort_keys=False))
|
||||
if verbose:
|
||||
print("Wrote " + app + " to " + composeFile)
|
||||
|
|
@ -85,7 +85,7 @@ def update(verbose: bool = False):
|
|||
|
||||
|
||||
def download(app: str = None):
|
||||
if(app is None):
|
||||
if app is None:
|
||||
apps = findAndValidateApps(appsDir)
|
||||
for app in apps:
|
||||
data = getAppYml(app)
|
||||
|
|
@ -117,7 +117,7 @@ def startInstalled():
|
|||
if os.path.isfile(userFile):
|
||||
with open(userFile, "r") as f:
|
||||
userData = json.load(f)
|
||||
threads = []
|
||||
#threads = []
|
||||
for app in userData["installedApps"]:
|
||||
print("Starting app {}...".format(app))
|
||||
# Run compose(args.app, "up --detach") asynchrounously for all apps, then exit(0) when all are finished
|
||||
|
|
@ -155,7 +155,7 @@ def getApp(appFile: str, appId: str):
|
|||
raise Exception("Error: Could not find metadata in " + appFile)
|
||||
app["metadata"]["id"] = appId
|
||||
|
||||
if('version' in app and str(app['version']) == "1"):
|
||||
if 'version' in app and str(app['version']) == "1":
|
||||
return createComposeConfigFromV1(app, nodeRoot)
|
||||
else:
|
||||
raise Exception("Error: Unsupported version of app.yml")
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ def getAppRegistry(apps, app_path):
|
|||
metadata['defaultPassword'] = metadata.get('defaultPassword', '')
|
||||
if metadata['defaultPassword'] == "$APP_SEED":
|
||||
metadata['defaultPassword'] = deriveEntropy("app-{}-seed".format(app))
|
||||
if("mainContainer" in metadata):
|
||||
if "mainContainer" in metadata:
|
||||
metadata.pop("mainContainer")
|
||||
app_metadata.append(metadata)
|
||||
return app_metadata
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ def validateApp(app: dict):
|
|||
with open(os.path.join(scriptDir, 'app-standard-v1.json'), 'r') as f:
|
||||
schemaVersion1 = json.loads(f.read())
|
||||
|
||||
if('version' in app and str(app['version']) == "1"):
|
||||
if 'version' in app and str(app['version']) == "1":
|
||||
try:
|
||||
validate(app, schemaVersion1)
|
||||
return True
|
||||
|
|
|
|||
2
scripts/configure
vendored
2
scripts/configure
vendored
|
|
@ -36,7 +36,7 @@ if not is_arm64() and not is_amd64():
|
|||
def is_compose_rc():
|
||||
try:
|
||||
output = subprocess.check_output(['docker', 'compose', 'version'])
|
||||
if(output.decode('utf-8').strip() == 'Docker Compose version v2.0.0-rc.3'):
|
||||
if output.decode('utf-8').strip() == 'Docker Compose version v2.0.0-rc.3':
|
||||
print("Using rc docker compose, updating...")
|
||||
return True
|
||||
else:
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user