heierbtc/citadel-core
1
0
Fork 0
forked from michael.heier/citadel-core
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor the app.yml compiler

Browse Source
This commit is contained in:
Aaron Dewes 2021-11-17 18:04:06 +00:00
parent 7b156dea7b
commit ca1a3e4d25
15 changed files with 411 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/app-manager.py
View File

@ -49,7 +49,7 @@ elif args.action == 'download':
updateRepos() updateRepos()
exit(0) exit(0)
elif args.action == 'update': elif args.action == 'update':
if(args.invoked_by_configure): if args.invoked_by_configure:
update(args.app) update(args.app)
else: else:
os.system(os.path.join(nodeRoot, "scripts", "configure")) os.system(os.path.join(nodeRoot, "scripts", "configure"))
@ -64,7 +64,7 @@ elif args.action == 'update':
elif args.action == 'update-online': elif args.action == 'update-online':
updateRepos() updateRepos()
print("Downloaded all updates") print("Downloaded all updates")
if(args.invoked_by_configure): if args.invoked_by_configure:
update(args.app) update(args.app)
else: else:
os.system(os.path.join(nodeRoot, "scripts", "configure")) os.system(os.path.join(nodeRoot, "scripts", "configure"))
@ -112,7 +112,7 @@ elif args.action == 'stop':
print("No app provided") print("No app provided")
exit(1) exit(1)
userData = getUserData() userData = getUserData()
if(args.app == "installed"): if args.app == "installed":
if "installedApps" in userData: if "installedApps" in userData:
stopInstalled() stopInstalled()
exit(0) exit(0)
@ -124,7 +124,7 @@ elif args.action == 'start':
exit(1) exit(1)
userData = getUserData() userData = getUserData()
if(args.app == "installed"): if args.app == "installed":
if "installedApps" in userData: if "installedApps" in userData:
startInstalled() startInstalled()
exit(0) exit(0)

4
app/app-standard-v1.json
View File

@ -186,6 +186,10 @@
"type": ["number", "array"] "type": ["number", "array"]
} }
} }
},
"restarts": {
"type": "string",
"description": "When the container should restart. Can be 'always' or 'on-failure'."
} }
}, },
"additionalProperties": false "additionalProperties": false

65
app/lib/citadelutils.py
View File

@ -4,6 +4,8 @@
import re import re
from click import types
# Helper functions # Helper functions
# Return a list of env vars in a string, supports both $NAM§ and ${NAME} format for the env var # Return a list of env vars in a string, supports both $NAM§ and ${NAME} format for the env var
# This can potentially be used to get around permissions, so this check is critical for security # This can potentially be used to get around permissions, so this check is critical for security
@ -42,20 +44,59 @@ def parse_dotenv(file_path):
exit(1) exit(1)
return envVars return envVars
# Combines two objects # Combines an object and a class
# If the key exists in both objects, the value of the second object is used # If the key exists in both objects, the value of the second object is used
# If the key does not exist in the first object, the value from the second object is used # If the key does not exist in the first object, the value from the second object is used
# If a key contains a list, the second object's list is appended to the first object's list # If a key contains a list, the second object's list is appended to the first object's list
# If a key contains another object, these objects are combined # If a key contains another object, these objects are combined
def combineObjects(obj1: dict, obj2: dict): def combineObjectAndClass(theClass, obj: dict):
for key in obj2: for key, value in obj.items():
if key in obj1: if key in theClass.__dict__:
if isinstance(obj1[key], list): if isinstance(value, list):
obj1[key] = obj1[key] + obj2[key] if isinstance(theClass.__dict__[key], list):
elif isinstance(obj1[key], dict): theClass.__dict__[key].extend(value)
obj1[key] = combineObjects(obj1[key], obj2[key])
else:
obj1[key] = obj2[key]
else: else:
obj1[key] = obj2[key] theClass.__dict__[key] = [theClass.__dict__[key]] + value
return obj1 elif isinstance(value, dict):
if isinstance(theClass.__dict__[key], dict):
theClass.__dict__[key].update(value)
else:
theClass.__dict__[key] = {theClass.__dict__[key]: value}
else:
theClass.__dict__[key] = value
else:
theClass.__dict__[key] = value
def is_builtin_type(obj):
return isinstance(obj, (int, float, str, bool, list, dict, types.ParamType))
# Convert a class to a dict
# Also strip any class member which is null or empty
def classToDict(theClass):
obj: dict = {}
for key, value in theClass.__dict__.items():
if value is None or (isinstance(value, list) and len(value) == 0):
continue
if isinstance(value, list):
for element in value:
newList = []
if is_builtin_type(element):
newList.append(element)
else:
newList.append(classToDict(element))
obj[key] = newList
elif isinstance(value, dict):
newDict = {}
for key, value in value.items():
if is_builtin_type(value):
newDict[key] = value
else:
newDict[key] = classToDict(value)
obj[key] = newDict
elif is_builtin_type(value):
obj[key] = value
else:
#print(value)
obj[key] = classToDict(value)
return obj

45
app/lib/composegenerator/shared/const.py
View File

@ -5,38 +5,39 @@
def permissions(): def permissions():
return { return {
"lnd": { "lnd": {
"environment_allow": { "environment_allow": [
"LND_IP": "${LND_IP}", "LND_IP",
"LND_GRPC_PORT": "${LND_GRPC_PORT}", "LND_GRPC_PORT",
"LND_REST_PORT": "${LND_REST_PORT}", "LND_REST_PORT",
"BITCOIN_NETWORK": "${BITCOIN_NETWORK}" "BITCOIN_NETWORK"
}, ],
"volumes": [ "volumes": [
'${LND_DATA_DIR}:/lnd:ro' '${LND_DATA_DIR}:/lnd:ro'
] ]
}, },
"bitcoind": { "bitcoind": {
"environment_allow": { "environment_allow": [
"BITCOIN_IP": "${BITCOIN_IP}", "BITCOIN_IP",
"BITCOIN_NETWORK": "${BITCOIN_NETWORK}", "BITCOIN_NETWORK",
"BITCOIN_P2P_PORT": "${BITCOIN_P2P_PORT}", "BITCOIN_P2P_PORT",
"BITCOIN_RPC_PORT": "${BITCOIN_RPC_PORT}", "BITCOIN_RPC_PORT",
"BITCOIN_RPC_USER": "${BITCOIN_RPC_USER}", "BITCOIN_RPC_USER",
"BITCOIN_RPC_PASS": "${BITCOIN_RPC_PASS}", "BITCOIN_RPC_PASS",
"BITCOIN_RPC_AUTH": "${BITCOIN_RPC_AUTH}", "BITCOIN_RPC_AUTH",
"BITCOIN_ZMQ_RAWBLOCK_PORT": "${BITCOIN_ZMQ_RAWBLOCK_PORT}", "BITCOIN_ZMQ_RAWBLOCK_PORT",
"BITCOIN_ZMQ_RAWTX_PORT": "${BITCOIN_ZMQ_RAWTX_PORT}", "BITCOIN_ZMQ_RAWTX_PORT",
"BITCOIN_ZMQ_HASHBLOCK_PORT": "${BITCOIN_ZMQ_HASHBLOCK_PORT}", "BITCOIN_ZMQ_HASHBLOCK_PORT",
}, ],
"volumes": [ "volumes": [
"${BITCOIN_DATA_DIR}:/bitcoin:ro" "${BITCOIN_DATA_DIR}:/bitcoin:ro"
] ]
}, },
"electrum": { "electrum": {
"environment_allow": { "environment_allow": [
"ELECTRUM_IP": "${ELECTRUM_IP}", "ELECTRUM_IP",
"ELECTRUM_PORT": "${ELECTRUM_PORT}", "ELECTRUM_PORT",
} ],
"volumes": []
} }
} }

27
app/lib/composegenerator/shared/env.py
View File

@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
import re import re
from lib.composegenerator.v1.types import App
from lib.composegenerator.shared.const import always_allowed_env from lib.composegenerator.shared.const import always_allowed_env
from lib.citadelutils import checkArrayContainsAllElements, getEnvVars from lib.citadelutils import checkArrayContainsAllElements, getEnvVars
@ -10,7 +11,7 @@ def validateEnvByValue(env: list, allowed: list, app_name: str):
# Combine always_allowed_env with allowed into one list # Combine always_allowed_env with allowed into one list
# Then check if all elements in env are in the resulting list # Then check if all elements in env are in the resulting list
all_allowed = allowed + always_allowed_env all_allowed = allowed + always_allowed_env
if(not checkArrayContainsAllElements(env, all_allowed)): if not checkArrayContainsAllElements(env, all_allowed):
# This has a weird syntax, and it confuses VSCode, but it works # This has a weird syntax, and it confuses VSCode, but it works
validation_regex = r"APP_{}(\S+)".format( validation_regex = r"APP_{}(\S+)".format(
app_name.upper().replace("-", "_")) app_name.upper().replace("-", "_"))
@ -23,24 +24,24 @@ def validateEnvByValue(env: list, allowed: list, app_name: str):
return True return True
def validateEnv(app: dict): def validateEnv(app: App):
# For every container of the app, check if all env vars in the strings in environment are defined in env # For every container of the app, check if all env vars in the strings in environment are defined in env
for container in app['containers']: for container in app.containers:
if 'environment' in container: if container.environment:
if 'environment_allow' in container: if container.environment_allow:
existingEnv = list(container['environment_allow'].keys()) existingEnv = container.environment_allow
del container['environment_allow'] del container.environment_allow
else: else:
existingEnv = [] existingEnv = []
# The next step depends on the type of the environment object, which is either a list or dict # The next step depends on the type of the environment object, which is either a list or dict
# If it's a list, split every string in it by the first=, then run getEnvVars(envVarValue) on it # If it's a list, split every string in it by the first=, then run getEnvVars(envVarValue) on it
# ON a dict, run getEnvVars(envVarValue) on every value of the environment object # ON a dict, run getEnvVars(envVarValue) on every value of the environment object
# Then check if all env vars returned by getEnvVars are defined in env # Then check if all env vars returned by getEnvVars are defined in env
if(isinstance(container['environment'], list)): if isinstance(container.environment, list):
raise Exception("List env vars are no longer supported for container {} of app {}".format( raise Exception("List env vars are no longer supported for container {} of app {}".format(
container['name'], app['metadata']['name'])) container.name, app.metadata.name))
elif(isinstance(container['environment'], dict)): elif isinstance(container.environment, dict):
for envVar in container['environment'].values(): for envVar in container.environment.values():
if(not validateEnvByValue(getEnvVars(envVar), existingEnv, app['metadata']['id'])): if not validateEnvByValue(getEnvVars(envVar), existingEnv, app.metadata.id):
raise Exception("Env vars not defined for container {} of app {}".format( raise Exception("Env vars not defined for container {} of app {}".format(
container['name'], app['metadata']['name'])) container.name, app.metadata.name))

85
app/lib/composegenerator/shared/main.py
View File

@ -3,64 +3,51 @@
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
# Main functions # Main functions
from lib.citadelutils import combineObjects from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container
from lib.composegenerator.shared.const import permissions from lib.composegenerator.shared.const import permissions
def convertContainerPermissions(app): def convertContainerPermissions(app: App) -> App:
for container in app['containers']: for container in app.containers:
if 'permissions' in container: for permission in container.permissions:
for permission in container['permissions']: if permission in permissions():
if(permission in permissions()): container.environment_allow.extend(permissions()[permission]['environment_allow'])
container = combineObjects( container.volumes.extend(permissions()[permission]['volumes'])
container, permissions()[permission]) else:
else: print("Warning: container {} of app {} defines unknown permission {}".format(container.name, app.metadata.name, permission))
print("Warning: container {} of app {} defines unknown permission {}".format(container['name'], app['metadata']['name'], permission))
return app return app
def convertContainersToServices(app: dict): def convertContainersToServices(app: AppStage3) -> AppStage3:
app['services'] = {} app.services = {}
for container in app['containers']: for container in app.containers:
if 'permissions' in container: if container.permissions:
del container['permissions'] del container.permissions
app['services'][container['name']] = container app.services[container.name] = container
del app['services'][container['name']]['name'] del app.services[container.name].name
del app['containers'] del app.containers
return app return app
# Converts the data of every container in app['containers'] to a volume, which is then added to the app # Converts the data of every container in app.containers to a volume, which is then added to the app
def convertDataDirToVolume(app: dict): def convertDataDirToVolume(app: App) -> AppStage2:
for container in app['containers']: for container in app.containers:
# Loop through data dirs in container['data'], if they don't contain a .., add them to container['volumes'] # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
# Also, a datadir shouldn't start with a / # Also, a datadir shouldn't start with a /
if 'data' in container: for dataDir in container.data:
for dataDir in container['data']: if dataDir.find("..") == -1 and dataDir[0] != "/":
if not 'volumes' in container: container.volumes.append(
container['volumes'] = [] '${APP_DATA_DIR}/' + dataDir)
if(dataDir.find("..") == -1 and dataDir[0] != "/"): else:
container['volumes'].append( print("Data dir " + dataDir +
'${APP_DATA_DIR}/' + dataDir) " contains invalid characters")
else: del container.data
print("Data dir " + dataDir + if container.bitcoin_mount_dir != None:
" contains invalid characters") if not 'bitcoind' in container.permissions:
del container['data'] print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
if 'bitcoin_mount_dir' in container:
if not 'permissions' in container or not 'bitcoind' in container['permissions']:
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container['name'], app['metadata']['name']))
# Skip this container # Skip this container
continue continue
if not 'volumes' in container: # Also skip the container if container.bitcoin_mount_dir contains a :
container['volumes'] = [] if container.bitcoin_mount_dir.find(":") == -1:
# Also skip the container if container['bitcoin_mount_dir'] contains a : container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir + ':ro')
if(container['bitcoin_mount_dir'].find(":") == -1): del container.bitcoin_mount_dir
container['volumes'].append('${BITCOIN_DATA_DIR}:' + container['bitcoin_mount_dir'] + ':ro')
del container['bitcoin_mount_dir']
return app return app
def addStopConfig(app: dict):
for container in app['containers']:
if not 'stop_grace_period' in container:
container['stop_grace_period'] = '1m'
container['restart'] = "on-failure"
return app

35
app/lib/composegenerator/v1/generate.py
View File

@ -2,30 +2,33 @@
# #
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
from lib.composegenerator.v1.types import App, AppStage4, generateApp
from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, addStopConfig, convertContainersToServices from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
from lib.composegenerator.shared.env import validateEnv from lib.composegenerator.shared.env import validateEnv
from lib.citadelutils import classToDict
import os import os
def createComposeConfigFromV1(app: dict, nodeRoot: str): def createComposeConfigFromV1(app: dict, nodeRoot: str):
if("version" in app): if "version" in app:
if(str(app['version']) != "1"): if str(app['version']) != "1":
print("Warning: app version is not supported") print("Warning: app version is not supported")
return False return False
envFile = os.path.join(nodeRoot, ".env") envFile = os.path.join(nodeRoot, ".env")
networkingFile = os.path.join(nodeRoot, "apps", "networking.json") networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
app = convertContainerPermissions(app) newApp: App = generateApp(app)
validateEnv(app) newApp = convertContainerPermissions(newApp)
app = convertDataDirToVolume(app) validateEnv(newApp)
app = configureIps(app, networkingFile, envFile) newApp = convertDataDirToVolume(newApp)
app = configureMainPort(app, nodeRoot) newApp = configureIps(newApp, networkingFile, envFile)
app = configureHiddenServices(app, nodeRoot) newApp = configureMainPort(newApp, nodeRoot)
app = addStopConfig(app) configureHiddenServices(newApp, nodeRoot)
app = convertContainersToServices(app) finalConfig: AppStage4 = convertContainersToServices(newApp)
del app['metadata'] newApp = classToDict(finalConfig)
if("version" in app): del newApp['metadata']
del app["version"] if "version" in newApp:
del newApp["version"]
# Set version to 3.8 (current compose file version) # Set version to 3.8 (current compose file version)
app = {'version': '3.8', **app} newApp = {'version': '3.8', **newApp}
return app return newApp

149
app/lib/composegenerator/v1/networking.py
View File

@ -2,6 +2,8 @@
# #
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
from dacite import from_dict
from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig
from lib.citadelutils import parse_dotenv from lib.citadelutils import parse_dotenv
import json import json
from os import path from os import path
@ -9,49 +11,51 @@ import random
from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService
def assignIp(container: dict, appId: str, networkingFile: str, envFile: str): def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
# Strip leading/trailing whitespace from container['name'] # Strip leading/trailing whitespace from container.name
container['name'] = container['name'].strip() container.name = container.name.strip()
# If the name still contains a newline, throw an error # If the name still contains a newline, throw an error
if(container['name'].find("\n") != -1): if container.name.find("\n") != -1:
raise Exception("Newline in container name") raise Exception("Newline in container name")
env_var = "APP_{}_{}_IP".format( env_var = "APP_{}_{}_IP".format(
appId.upper().replace("-", "_"), appId.upper().replace("-", "_"),
container['name'].upper().replace("-", "_") container.name.upper().replace("-", "_")
) )
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
# can be used # can be used
usedIps = [] usedIps = []
networkingData = {} networkingData = {}
if(path.isfile(networkingFile)): if path.isfile(networkingFile):
with open(networkingFile, 'r') as f: with open(networkingFile, 'r') as f:
networkingData = json.load(f) networkingData = json.load(f)
if('ip_addresses' in networkingData): if 'ip_addresses' in networkingData:
usedIps = list(networkingData['ip_addresses'].values()) usedIps = list(networkingData['ip_addresses'].values())
else: else:
networkingData['ip_addresses'] = {} networkingData.ip_addresses = {}
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
# If the IP is already in use, it will be tried again until it's not in use # If the IP is already in use, it will be tried again until it's not in use
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile # If it's not in use, it will be added to the usedIps list and written to the usedIpFile
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive), # If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
# Throw an error, because no more IPs can be used # Throw an error, because no more IPs can be used
if(len(usedIps) == 235): if len(usedIps) == 235:
raise Exception("No more IPs can be used") raise Exception("No more IPs can be used")
if("{}-{}".format(appId, container['name']) in networkingData['ip_addresses']): if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
ip = networkingData['ip_addresses']["{}-{}".format(appId, container['name'])] ip = networkingData['ip_addresses']["{}-{}".format(
appId, container.name)]
else: else:
while True: while True:
ip = "10.21.21." + str(random.randint(20, 255)) ip = "10.21.21." + str(random.randint(20, 255))
if(ip not in usedIps): if ip not in usedIps:
networkingData['ip_addresses']["{}-{}".format(appId, container['name'])] = ip networkingData['ip_addresses']["{}-{}".format(
appId, container.name)] = ip
break break
container['networks'] = {'default': { container.networks = from_dict(data_class=NetworkConfig, data={'default': {
'ipv4_address': "$" + env_var}} 'ipv4_address': "$" + env_var}})
dotEnv = parse_dotenv(envFile) dotEnv = parse_dotenv(envFile)
if(env_var in dotEnv and str(dotEnv[env_var]) == str(ip)): if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
return container return container
# Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile # Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile
@ -63,21 +67,21 @@ def assignIp(container: dict, appId: str, networkingFile: str, envFile: str):
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str): def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
# Strip leading/trailing whitespace from container['name'] # Strip leading/trailing whitespace from container.name
container['name'] = container['name'].strip() container.name = container.name.strip()
# If the name still contains a newline, throw an error # If the name still contains a newline, throw an error
if(container['name'].find("\n") != -1 or container['name'].find(" ") != -1): if container.name.find("\n") != -1 or container.name.find(" ") != -1:
raise Exception("Newline or space in container name") raise Exception("Newline or space in container name")
env_var = "APP_{}_{}_PORT".format( env_var = "APP_{}_{}_PORT".format(
appId.upper().replace("-", "_"), appId.upper().replace("-", "_"),
container['name'].upper().replace("-", "_") container.name.upper().replace("-", "_")
) )
port = getFreePort(networkingFile, appId) port = getFreePort(networkingFile, appId)
dotEnv = parse_dotenv(envFile) dotEnv = parse_dotenv(envFile)
if(env_var in dotEnv and str(dotEnv[env_var]) == str(port)): if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
return {"port": port, "env_var": "${{{}}}".format(env_var)} return {"port": port, "env_var": "${{{}}}".format(env_var)}
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile # Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
@ -88,32 +92,34 @@ def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
# where the outer {{ }} will be replaced by {} in the returned string # where the outer {{ }} will be replaced by {} in the returned string
return {"port": port, "env_var": "${{{}}}".format(env_var)} return {"port": port, "env_var": "${{{}}}".format(env_var)}
def getMainContainer(app: dict):
if len(app['containers']) == 1:
return app['containers'][0]
else:
if not 'mainContainer' in app['metadata']:
app['metadata']['mainContainer'] = 'main'
for container in app['containers']:
if container['name'] == app['metadata']['mainContainer']:
return container
raise Exception("No main container found for app {}".format(app['metadata']['name']))
def configureMainPort(app: dict, nodeRoot: str): def getMainContainer(app: dict):
if len(app.containers) == 1:
return app.containers[0]
else:
if not app.metadata.mainContainer:
app.metadata.mainContainer = 'main'
for container in app.containers:
if container.name == app.metadata.mainContainer:
return container
raise Exception(
"No main container found for app {}".format(app.metadata.name))
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
registryFile = path.join(nodeRoot, "apps", "registry.json") registryFile = path.join(nodeRoot, "apps", "registry.json")
registry: list = [] registry: list = []
if(path.isfile(registryFile)): if path.isfile(registryFile):
with open(registryFile, 'r') as f: with open(registryFile, 'r') as f:
registry = json.load(f) registry = json.load(f)
else: else:
raise Exception("Registry file not found") raise Exception("Registry file not found")
dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
mainContainer = getMainContainer(app) mainContainer = getMainContainer(app)
portDetails = assignPort(mainContainer, app['metadata']['id'], path.join( portDetails = assignPort(mainContainer, app.metadata.id, path.join(
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
containerPort = portDetails['port'] containerPort = portDetails['port']
portAsEnvVar = portDetails['env_var'] portAsEnvVar = portDetails['env_var']
@ -121,37 +127,37 @@ def configureMainPort(app: dict, nodeRoot: str):
mainPort = False mainPort = False
if "port" in mainContainer: if mainContainer.port:
portToAppend = "{}:{}".format(portAsEnvVar, mainContainer['port']) portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port)
mainPort = mainContainer['port'] mainPort = mainContainer.port
del mainContainer['port'] del mainContainer.port
else: else:
portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar) portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar)
if "ports" in mainContainer: if mainContainer.ports:
mainContainer['ports'].append(portToAppend) mainContainer.ports.append(portToAppend)
# Set the main port to the first port in the list, if it contains a :, it's the port after the : # Set the main port to the first port in the list, if it contains a :, it's the port after the :
# If it doesn't contain a :, it's the port itself # If it doesn't contain a :, it's the port itself
if mainPort == False: if mainPort == False:
mainPort = mainContainer['ports'][0] mainPort = mainContainer.ports[0]
if(mainPort.find(":") != -1): if mainPort.find(":") != -1:
mainPort = mainPort.split(":")[1] mainPort = mainPort.split(":")[1]
else: else:
mainContainer['ports'] = [portToAppend] mainContainer.ports = [portToAppend]
if mainPort == False: if mainPort == False:
mainPort = portDetails['port'] mainPort = portDetails['port']
mainContainer = assignIp(mainContainer, app['metadata']['id'], path.join( mainContainer = assignIp(mainContainer, app.metadata.id, path.join(
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env")) nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
# If the IP wasn't in dotenv before, now it should be # If the IP wasn't in dotenv before, now it should be
dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
containerIP = dotEnv['APP_{}_{}_IP'.format(app['metadata']['id'].upper().replace( containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace(
"-", "_"), mainContainer['name'].upper().replace("-", "_"))] "-", "_"), mainContainer.name.upper().replace("-", "_"))]
hiddenservice = getHiddenService( hiddenservice = getHiddenService(
app['metadata']['name'], app['metadata']['id'], containerIP, mainPort) app.metadata.name, app.metadata.id, containerIP, mainPort)
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
@ -159,10 +165,10 @@ def configureMainPort(app: dict, nodeRoot: str):
f.write(hiddenservice) f.write(hiddenservice)
# Also set the port in metadata # Also set the port in metadata
app['metadata']['port'] = int(containerPort) app.metadata.port = int(containerPort)
for registryApp in registry: for registryApp in registry:
if(registryApp['id'] == app['metadata']['id']): if registryApp['id'] == app.metadata.id:
registry[registry.index(registryApp)]['port'] = int(containerPort) registry[registry.index(registryApp)]['port'] = int(containerPort)
break break
@ -172,48 +178,49 @@ def configureMainPort(app: dict, nodeRoot: str):
return app return app
def configureIps(app: dict, networkingFile: str, envFile: str): def configureIps(app: AppStage2, networkingFile: str, envFile: str):
for container in app['containers']: for container in app.containers:
if('noNetwork' in container and container['noNetwork']): if container.noNetwork:
# Check if port is defined for the container # Check if port is defined for the container
if('port' in container): if container.port:
raise Exception("Port defined for container without network") raise Exception("Port defined for container without network")
if(app['metadata']['mainContainer'] == container['name']): if app.metadata.mainContainer == container.name:
raise Exception("Main container without network") raise Exception("Main container without network")
# Skip this iteration of the loop # Skip this iteration of the loop
continue continue
container = assignIp(container, app['metadata']['id'], networkingFile, envFile) container = assignIp(container, app.metadata.id,
networkingFile, envFile)
return app return app
def configureHiddenServices(app: dict, nodeRoot: str):
def configureHiddenServices(app: dict, nodeRoot: str) -> None:
dotEnv = parse_dotenv(path.join(nodeRoot, ".env")) dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
hiddenServices = "" hiddenServices = ""
if len(app['containers']) == 1: if len(app.containers) == 1:
mainContainer = app['containers'][0] mainContainer = app.containers[0]
else: else:
mainContainer = None mainContainer = None
if not 'mainContainer' in app['metadata']: if app.metadata.mainContainer == None:
app['metadata']['mainContainer'] = 'main' app.metadata.mainContainer = 'main'
for container in app['containers']: for container in app.containers:
if container['name'] == app['metadata']['mainContainer']: if container.name == app.metadata.mainContainer:
mainContainer = container mainContainer = container
break break
if mainContainer is None: if mainContainer is None:
raise Exception("No main container found") raise Exception("No main container found")
for container in app['containers']: for container in app.containers:
env_var = "APP_{}_{}_IP".format( env_var = "APP_{}_{}_IP".format(
app["metadata"]["id"].upper().replace("-", "_"), app.metadata.id.upper().replace("-", "_"),
container['name'].upper().replace("-", "_") container.name.upper().replace("-", "_")
) )
hiddenServices += getContainerHiddenService(app["metadata"]["name"], app["metadata"]["id"], container, dotEnv[env_var], container["name"] == mainContainer["name"]) hiddenServices += getContainerHiddenService(
app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name)
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"] torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)] torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f: with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(hiddenServices) f.write(hiddenServices)
return app

151
app/lib/composegenerator/v1/types.py Normal file
View File

@ -0,0 +1,151 @@
from typing import Union
from dataclasses import dataclass, field
from dacite import from_dict
@dataclass
class Metadata:
id: str
name: str
version: str
category: str
tagline: str
description: str
developer: str
website: str
repo: str
support: str
gallery: list[str] = field(default_factory=list)
dependencies: list[str] = field(default_factory=list)
mainContainer: Union[str, None] = None
updateContainer: Union[str, None] = None
path: str = ""
defaultPassword: str = ""
torOnly: bool = False
@dataclass
class Container:
name: str
image: str
permissions: list = field(default_factory=list)
ports: list = field(default_factory=list)
port: Union[int, None] = None
environment: Union[dict, None] = None
data: list = field(default_factory=list)
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: list = field(default_factory=list)
entrypoint: Union[list[str], str] = field(default_factory=list)
bitcoin_mount_dir: Union[str, None] = None
command: Union[list[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
environment_allow: list = field(default_factory=list)
# Only added later
volumes: list = field(default_factory=list)
restarts: Union[str, None] = None
@dataclass
class App:
version: Union[str, int]
metadata: Metadata
containers: list[Container]
# Generate an app instance from an app dict
def generateApp(appDict):
return from_dict(data_class=App, data=appDict)
@dataclass
class Network:
ipv4_address: Union[str, None] = None
@dataclass
class NetworkConfig:
default: Network
# After converting data dir and defining volumes, stage 2
@dataclass
class ContainerStage2:
id: str
name: str
image: str
permissions: list[str] = field(default_factory=list)
ports: list = field(default_factory=list)
environment: Union[dict, None] = None
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: list[str] = field(default_factory=list)
entrypoint: Union[list[str], str] = field(default_factory=list)
command: Union[list[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
volumes: list[str] = field(default_factory=list)
networks: NetworkConfig = field(default_factory=NetworkConfig)
restarts: Union[str, None] = None
@dataclass
class AppStage2:
version: Union[str, int]
metadata: Metadata
containers: list[ContainerStage2]
@dataclass
class MetadataStage3:
id: str
name: str
version: str
category: str
tagline: str
description: str
developer: str
website: str
dependencies: list[str]
repo: str
support: str
gallery: list[str]
mainContainer: Union[str, None] = None
updateContainer: Union[str, None] = None
path: str = ""
defaultPassword: str = ""
torOnly: bool = False
@dataclass
class AppStage3:
version: Union[str, int]
metadata: MetadataStage3
containers: list[ContainerStage2]
@dataclass
class ContainerStage4:
id: str
name: str
image: str
ports: list = field(default_factory=list)
environment: Union[dict, None] = None
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: list[str] = field(default_factory=list)
entrypoint: Union[list[str], str] = field(default_factory=list)
command: Union[list[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
volumes: list[str] = field(default_factory=list)
networks: NetworkConfig = field(default_factory=NetworkConfig)
restarts: Union[str, None] = None
@dataclass
class AppStage4:
version: Union[str, int]
metadata: MetadataStage3
services: list[ContainerStage4]

40
app/lib/composegenerator/v1/utils/networking.py
View File

@ -6,6 +6,8 @@ import json
import os import os
import random import random
from lib.composegenerator.v1.types import Container
def getFreePort(networkingFile: str, appId: str): def getFreePort(networkingFile: str, appId: str):
# Ports used currently in Citadel # Ports used currently in Citadel
@ -13,22 +15,22 @@ def getFreePort(networkingFile: str, appId: str):
usedPorts = [80, 8333, 8332, 28332, 28333, 28334, 10009, 8080, 50001, 9050, 3002, 3000, 3300, 3001, 3004, 25441, usedPorts = [80, 8333, 8332, 28332, 28333, 28334, 10009, 8080, 50001, 9050, 3002, 3000, 3300, 3001, 3004, 25441,
3003, 3007, 3006, 3009, 3005, 8898, 3008, 8081, 8082, 8083, 8085, 2222, 8086, 8087, 8008, 8088, 8089, 8091] 3003, 3007, 3006, 3009, 3005, 8898, 3008, 8081, 8082, 8083, 8085, 2222, 8086, 8087, 8008, 8088, 8089, 8091]
networkingData = {} networkingData = {}
if(os.path.isfile(networkingFile)): if os.path.isfile(networkingFile):
with open(networkingFile, 'r') as f: with open(networkingFile, 'r') as f:
networkingData = json.load(f) networkingData = json.load(f)
if('ports' in networkingData): if 'ports' in networkingData:
usedPorts += list(networkingData['ports'].values()) usedPorts += list(networkingData['ports'].values())
else: else:
networkingData['ports'] = {} networkingData['ports'] = {}
if(appId in networkingData['ports']): if appId in networkingData['ports']:
return networkingData['ports'][appId] return networkingData['ports'][appId]
while True: while True:
port = str(random.randint(1024, 49151)) port = str(random.randint(1024, 49151))
if(port not in usedPorts): if port not in usedPorts:
# Check if anyhing is listening on the specific port # Check if anyhing is listening on the specific port
if(os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0): if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
networkingData['ports'][appId] = port networkingData['ports'][appId] = port
break break
@ -63,21 +65,21 @@ def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str:
return getHiddenServiceString(appName, appId, appPort, appIp, "80") return getHiddenServiceString(appName, appId, appPort, appIp, "80")
def getContainerHiddenService(appName: str, appId: str, container: dict, containerIp: str, isMainContainer: bool) -> str: def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str:
if not "needsHiddenService" in container and not isMainContainer: if not container.needsHiddenService and not isMainContainer:
return "" return ""
if ("ports" in container or not "port" in container) and not "hiddenServicePort" in container and not isMainContainer: if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer:
print("Container {} for app {} isn't compatible with hidden service assignment".format( print("Container {} for app {} isn't compatible with hidden service assignment".format(
container["name"], appName)) container.name, appName))
return "" return ""
if isMainContainer: if isMainContainer:
if not "hiddenServicePorts" in container: if not container.hiddenServicePorts:
return "" return ""
# hiddenServicePorts is a map of hidden service name to port # hiddenServicePorts is a map of hidden service name to port
# We need to generate a hidden service for each one # We need to generate a hidden service for each one
hiddenServices = "" hiddenServices = ""
for name, port in container["hiddenServicePorts"].items(): for name, port in container.hiddenServicePorts.items():
if ".." in name: if ".." in name:
print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName)) print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName))
@ -88,15 +90,15 @@ def getContainerHiddenService(appName: str, appId: str, container: dict, contain
else: else:
hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format( hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format(
appId, name), port, containerIp, port) appId, name), port, containerIp, port)
del container["hiddenServicePorts"] del container.hiddenServicePorts
return hiddenServices return hiddenServices
del container["needsHiddenService"] del container.needsHiddenService
if not "port" in container: if not container.port:
data = getHiddenServiceString(appName + container["name"], "{}-{}".format( data = getHiddenServiceString(appName + container.name, "{}-{}".format(
appId, container["name"]), container["hiddenServicePort"], containerIp, "80") appId, container.name), container.hiddenServicePort, containerIp, "80")
del container["hiddenServicePort"] del container.hiddenServicePort
return data return data
else: else:
return getHiddenServiceString(appName + container["name"], "{}-{}".format( return getHiddenServiceString(appName + container.name, "{}-{}".format(
appId, container["name"]), container["port"], containerIp, container["port"]) appId, container.name), container.port, containerIp, container.port)

2
app/lib/entropy.py
View File

@ -12,7 +12,7 @@ def deriveEntropy(identifier: str):
seedFile = os.path.join(nodeRoot, "db", "citadel-seed", "seed") seedFile = os.path.join(nodeRoot, "db", "citadel-seed", "seed")
alternativeSeedFile = os.path.join(nodeRoot, "db", "citadel-seed", "seed") alternativeSeedFile = os.path.join(nodeRoot, "db", "citadel-seed", "seed")
if not os.path.isfile(seedFile): if not os.path.isfile(seedFile):
if(os.path.isfile(alternativeSeedFile)): if os.path.isfile(alternativeSeedFile):
seedFile = alternativeSeedFile seedFile = alternativeSeedFile
else: else:
print("No seed file found, exiting...") print("No seed file found, exiting...")

8
app/lib/manage.py
View File

@ -77,7 +77,7 @@ def update(verbose: bool = False):
appYml = os.path.join(appsDir, app, "app.yml") appYml = os.path.join(appsDir, app, "app.yml")
with open(composeFile, "w") as f: with open(composeFile, "w") as f:
appCompose = getApp(appYml, app) appCompose = getApp(appYml, app)
if(appCompose): if appCompose:
f.write(yaml.dump(appCompose, sort_keys=False)) f.write(yaml.dump(appCompose, sort_keys=False))
if verbose: if verbose:
print("Wrote " + app + " to " + composeFile) print("Wrote " + app + " to " + composeFile)
@ -85,7 +85,7 @@ def update(verbose: bool = False):
def download(app: str = None): def download(app: str = None):
if(app is None): if app is None:
apps = findAndValidateApps(appsDir) apps = findAndValidateApps(appsDir)
for app in apps: for app in apps:
data = getAppYml(app) data = getAppYml(app)
@ -117,7 +117,7 @@ def startInstalled():
if os.path.isfile(userFile): if os.path.isfile(userFile):
with open(userFile, "r") as f: with open(userFile, "r") as f:
userData = json.load(f) userData = json.load(f)
threads = [] #threads = []
for app in userData["installedApps"]: for app in userData["installedApps"]:
print("Starting app {}...".format(app)) print("Starting app {}...".format(app))
# Run compose(args.app, "up --detach") asynchrounously for all apps, then exit(0) when all are finished # Run compose(args.app, "up --detach") asynchrounously for all apps, then exit(0) when all are finished
@ -155,7 +155,7 @@ def getApp(appFile: str, appId: str):
raise Exception("Error: Could not find metadata in " + appFile) raise Exception("Error: Could not find metadata in " + appFile)
app["metadata"]["id"] = appId app["metadata"]["id"] = appId
if('version' in app and str(app['version']) == "1"): if 'version' in app and str(app['version']) == "1":
return createComposeConfigFromV1(app, nodeRoot) return createComposeConfigFromV1(app, nodeRoot)
else: else:
raise Exception("Error: Unsupported version of app.yml") raise Exception("Error: Unsupported version of app.yml")

2
app/lib/metadata.py
View File

@ -36,7 +36,7 @@ def getAppRegistry(apps, app_path):
metadata['defaultPassword'] = metadata.get('defaultPassword', '') metadata['defaultPassword'] = metadata.get('defaultPassword', '')
if metadata['defaultPassword'] == "$APP_SEED": if metadata['defaultPassword'] == "$APP_SEED":
metadata['defaultPassword'] = deriveEntropy("app-{}-seed".format(app)) metadata['defaultPassword'] = deriveEntropy("app-{}-seed".format(app))
if("mainContainer" in metadata): if "mainContainer" in metadata:
metadata.pop("mainContainer") metadata.pop("mainContainer")
app_metadata.append(metadata) app_metadata.append(metadata)
return app_metadata return app_metadata

2
app/lib/validate.py
View File

@ -15,7 +15,7 @@ def validateApp(app: dict):
with open(os.path.join(scriptDir, 'app-standard-v1.json'), 'r') as f: with open(os.path.join(scriptDir, 'app-standard-v1.json'), 'r') as f:
schemaVersion1 = json.loads(f.read()) schemaVersion1 = json.loads(f.read())
if('version' in app and str(app['version']) == "1"): if 'version' in app and str(app['version']) == "1":
try: try:
validate(app, schemaVersion1) validate(app, schemaVersion1)
return True return True

2
scripts/configure vendored
View File

@ -36,7 +36,7 @@ if not is_arm64() and not is_amd64():
def is_compose_rc(): def is_compose_rc():
try: try:
output = subprocess.check_output(['docker', 'compose', 'version']) output = subprocess.check_output(['docker', 'compose', 'version'])
if(output.decode('utf-8').strip() == 'Docker Compose version v2.0.0-rc.3'): if output.decode('utf-8').strip() == 'Docker Compose version v2.0.0-rc.3':
print("Using rc docker compose, updating...") print("Using rc docker compose, updating...")
return True return True
else: else: