Prepare to move to new app CLI

This commit is contained in:
AaronDewes 2022-10-28 10:23:24 +00:00
parent 878fb52791
commit 6ca5fe89cf
7 changed files with 8 additions and 460 deletions

View File

@ -11,7 +11,6 @@ import os
from lib.manage import (compose, createDataDir, deleteData, deriveEntropy,
download, getAvailableUpdates, getUserData,
setInstalled, setRemoved, update, updateRepos)
from lib.validate import findAndValidateApps
# Print an error if user is not root
if os.getuid() != 0:
@ -28,10 +27,7 @@ legacyScript = os.path.join(nodeRoot, "scripts", "app")
parser = argparse.ArgumentParser(description="Manage apps on your Citadel")
parser.add_argument('action', help='What to do with the app database.', choices=[
"list", "download", "generate", "update", "list-updates", "ls-installed", "install", "uninstall", "stop", "start", "compose", "restart", "entropy"])
# Add the --invoked-by-configure option, which is hidden from the user in --help
parser.add_argument('--invoked-by-configure',
action='store_true', help=argparse.SUPPRESS)
"download", "generate", "update", "list-updates", "ls-installed", "install", "uninstall", "stop", "start", "compose", "restart", "entropy"])
parser.add_argument('--verbose', '-v', action='store_true')
parser.add_argument(
'app', help='Optional, the app to perform an action on. (For install, uninstall, stop, start and compose)', nargs='?')
@ -43,12 +39,7 @@ args = parser.parse_args()
if args.action is None:
args.action = 'list'
if args.action == 'list':
apps = findAndValidateApps(appsDir)
for app in apps:
print(app)
exit(0)
elif args.action == "list-updates":
if args.action == "list-updates":
availableUpdates = getAvailableUpdates()
print(json.dumps(availableUpdates))
exit(0)
@ -56,17 +47,7 @@ elif args.action == 'download':
updateRepos()
exit(0)
elif args.action == 'generate':
if args.invoked_by_configure:
update(args.app)
else:
os.system(os.path.join(nodeRoot, "scripts", "configure"))
os.chdir(nodeRoot)
os.system("docker compose stop app-tor")
os.system("docker compose start app-tor")
os.system("docker compose stop app-2-tor")
os.system("docker compose start app-2-tor")
os.system("docker compose stop app-3-tor")
os.system("docker compose start app-3-tor")
update(args.app)
exit(0)
elif args.action == 'update':
if args.app is None:
@ -75,17 +56,7 @@ elif args.action == 'update':
else:
download(args.app)
print("Downloaded latest {} version".format(args.app))
if args.invoked_by_configure:
update(args.verbose)
else:
os.system(os.path.join(nodeRoot, "scripts", "configure"))
os.chdir(nodeRoot)
os.system("docker compose stop app-tor")
os.system("docker compose start app-tor")
os.system("docker compose stop app-2-tor")
os.system("docker compose start app-2-tor")
os.system("docker compose stop app-3-tor")
os.system("docker compose start app-3-tor")
update(args.verbose)
exit(0)
elif args.action == 'ls-installed':
# Load the userFile as JSON, check if installedApps is in it, and if so, print the apps

View File

@ -21,8 +21,6 @@ import semver
import yaml
from lib.citadelutils import FileLock, parse_dotenv
from lib.entropy import deriveEntropy
from lib.metadata import getAppMetadata
from lib.validate import findAndValidateApps
# For an array of threads, join them and wait for them to finish
@ -88,45 +86,6 @@ def convert_to_upper(string):
def replace_vars(file_content: str):
return re.sub(r'<(.*?)>', lambda m: get_var(convert_to_upper(m.group(1))), file_content)
def handleAppV3OrV4(app):
# Currently part of Citadel
services = ["lnd", "bitcoind"]
userData = getUserData()
if not "installedApps" in userData:
userData["installedApps"] = []
services.extend(userData["installedApps"])
services.extend(getInstalledVirtualApps())
composeFile = os.path.join(appsDir, app, "docker-compose.yml")
os.chown(os.path.join(appsDir, app), 1000, 1000)
if not os.path.isfile(os.path.join(appsDir, app, "result.yml")):
os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli convert --app-name '{}' --port-map /apps/ports.json --services '{}' /apps/{}/app.yml /apps/{}/result.yml".format(appsDir, dependencies['app-cli'], app, ",".join(services), app, app))
with open(os.path.join(appsDir, app, "result.yml"), "r") as resultFile:
resultYml = yaml.safe_load(resultFile)
with open(composeFile, "w") as dockerComposeFile:
yaml.dump(resultYml["spec"], dockerComposeFile)
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(os.path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(replace_vars(resultYml["new_tor_entries"]))
registryFile = os.path.join(nodeRoot, "apps", "registry.json")
registry: list = []
lock = FileLock("citadel_registry_lock", dir="/tmp")
lock.acquire()
if os.path.isfile(registryFile):
with open(registryFile, 'r') as f:
registry = json.load(f)
resultYml["metadata"]['port'] = resultYml["port"]
resultYml["metadata"]['defaultPassword'] = resultYml["metadata"].get('defaultPassword', '')
if resultYml["metadata"]['defaultPassword'] == "$APP_SEED":
resultYml["metadata"]['defaultPassword'] = deriveEntropy("app-{}-seed".format(app))
registry.append(resultYml["metadata"])
with open(registryFile, 'w') as f:
json.dump(registry, f, indent=4, sort_keys=True)
lock.release()
def getAppYml(name):
with open(os.path.join(appsDir, "sourceMap.json"), "r") as f:
@ -147,44 +106,7 @@ def getAppYml(name):
return False
def update(verbose: bool = False):
apps = findAndValidateApps(appsDir)
portCache = {}
try:
with open(os.path.join(appsDir, "ports.cache.json"), "w") as f:
portCache = json.load(f)
except Exception: pass
registry = getAppMetadata(apps, appsDir, portCache)
with open(os.path.join(appsDir, "ports.json"), "w") as f:
json.dump(registry["ports"], f, sort_keys=True)
with open(os.path.join(appsDir, "ports.cache.json"), "w") as f:
json.dump(registry["portCache"], f, sort_keys=True)
with open(os.path.join(appsDir, "virtual-apps.json"), "w") as f:
json.dump(registry["virtual_apps"], f, sort_keys=True)
print("Processed app metadata")
# Delete the registry so it's regenerated
os.remove(os.path.join(nodeRoot, "apps", "registry.json"))
os.system("docker pull {}".format(dependencies['app-cli']))
threads = list()
# Loop through the apps and generate valid compose files from them, then put these into the app dir
for app in apps:
try:
appYml = os.path.join(appsDir, app, "app.yml")
with open(appYml, 'r') as f:
appDefinition = yaml.safe_load(f)
if ('citadel_version' in appDefinition) or ('version' in appDefinition and str(appDefinition['version']) == "3"):
thread = threading.Thread(target=handleAppV3OrV4, args=(app,))
thread.start()
threads.append(thread)
else:
raise Exception("Error: Unsupported version of app.yml")
except Exception as err:
print("Failed to convert app {}".format(app))
print(traceback.format_exc())
joinThreads(threads)
os.system("docker run --rm -v {}:/citadel -u 1000:1000 {} /app-cli convert /citadel".format(nodeRoot, dependencies['app-cli']))
print("Generated configuration successfully")

View File

@ -1,242 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
import json
import os
import random
import traceback
from os import path
import yaml
from lib.citadelutils import parse_dotenv
from lib.entropy import deriveEntropy
appPorts = {}
appPortMap = {}
disabledApps = []
def assignIpV4(appId: str, containerName: str):
scriptDir = path.dirname(path.realpath(__file__))
nodeRoot = path.join(scriptDir, "..", "..")
networkingFile = path.join(nodeRoot, "apps", "networking.json")
envFile = path.join(nodeRoot, ".env")
cleanContainerName = containerName.strip()
# If the name still contains a newline, throw an error
if cleanContainerName.find("\n") != -1:
raise Exception("Newline in container name")
env_var = "APP_{}_{}_IP".format(
appId.upper().replace("-", "_"),
cleanContainerName.upper().replace("-", "_")
)
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
# can be used
usedIps = []
networkingData = {}
if path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ip_addresses' in networkingData:
usedIps = list(networkingData['ip_addresses'].values())
else:
networkingData['ip_addresses'] = {}
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
# If the IP is already in use, it will be tried again until it's not in use
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
# Throw an error, because no more IPs can be used
if len(usedIps) == 235:
raise Exception("No more IPs can be used")
if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']:
ip = networkingData['ip_addresses']["{}-{}".format(
appId, cleanContainerName)]
else:
while True:
ip = "10.21.21." + str(random.randint(20, 255))
if ip not in usedIps:
networkingData['ip_addresses']["{}-{}".format(
appId, cleanContainerName)] = ip
break
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
return
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, ip))
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
def appPortsToMap():
for port in appPorts:
appId = appPorts[port]["app"]
containerId = appPorts[port]["container"]
realPort = appPorts[port]["port"]
if not appId in appPortMap:
appPortMap[appId] = {}
if not containerId in appPortMap[appId]:
appPortMap[appId][containerId] = []
appPortMap[appId][containerId].append({
"publicPort": port,
"internalPort": realPort,
"dynamic": appPorts[port]["dynamic"]
})
# For every app, parse the app.yml in ../apps/[name] and
# check their metadata, and return a list of all app's metadata
# Also check the path and defaultPassword and set them to an empty string if they don't exist
# In addition, set id on the metadata to the name of the app
# Return a list of all app's metadata
def getAppMetadata(apps, app_path, portCache):
virtual_apps = {}
appPorts = portCache
for app in apps:
app_yml_path = os.path.join(app_path, app, 'app.yml')
if os.path.isfile(app_yml_path):
try:
with open(app_yml_path, 'r') as f:
app_yml = yaml.safe_load(f.read())
version = False
if 'version' in app_yml:
version = int(app_yml['version'])
elif 'citadel_version' in app_yml:
version = int(app_yml['citadel_version'])
metadata: dict = app_yml['metadata']
if "implements" in metadata:
implements = metadata["implements"]
if implements not in virtual_apps:
virtual_apps[implements] = []
virtual_apps[implements].append(app)
if version == 3:
getPortsV3App(app_yml, app)
elif version == 4:
getPortsV4App(app_yml, app)
except Exception as e:
print(traceback.format_exc())
print("App {} is invalid!".format(app))
appPortsToMap()
return {
"virtual_apps": virtual_apps,
"ports": appPortMap,
"portCache": appPorts,
}
citadelPorts = [
# Dashboard
80,
# Sometimes used by nginx with some setups
433,
# Dashboard SSL
443,
# Bitcoin Core P2P
8333,
# LND gRPC
10009,
# LND REST
8080,
# Electrum Server
# 50001,
# Tor Proxy
9050,
]
lastPort = 3000
def getNewPort(usedPorts, appId, containerName, allowExisting):
lastPort2 = lastPort
while lastPort2 in usedPorts.keys() or lastPort2 in citadelPorts:
if allowExisting and lastPort2 in usedPorts.keys() and usedPorts[lastPort2]["app"] == appId and usedPorts[lastPort2]["container"] == containerName:
break
lastPort2 = lastPort2 + 1
return lastPort2
def validatePort(containerName, port, appId, priority: int, isDynamic = False, implements = ""):
if port not in appPorts and port not in citadelPorts and port != 0:
appPorts[port] = {
"app": appId,
"port": port,
"container": containerName,
"priority": priority,
"dynamic": isDynamic,
"implements": implements,
}
else:
if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != containerName:
if port in appPorts and priority > appPorts[port]["priority"]:
#print("Prioritizing app {} over {}".format(appId, appPorts[port]["app"]))
newPort = getNewPort(appPorts, appPorts[port]["app"], appPorts[port]["container"], False)
appPorts[newPort] = appPorts[port].copy()
appPorts[port] = {
"app": appId,
"port": port,
"container": containerName,
"priority": priority,
"dynamic": isDynamic,
"implements": implements,
}
else:
# Apps implement the same service and can't be installed together, so we an safely ignore a port conflict
if port in appPorts and implements != "" and implements == appPorts[port]["implements"]:
return
if priority == 2:
disabledApps.append(appId)
print("App {} disabled because of port conflict".format(appId))
else:
newPort = getNewPort(appPorts, appId, containerName, True)
internalPort = port
if isDynamic:
internalPort = newPort
#print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, containerName, newPort, port))
appPorts[newPort] = {
"app": appId,
"port": internalPort,
"container": containerName,
"priority": priority,
"dynamic": isDynamic,
"implements": implements,
}
def getPortsV3App(app, appId):
for appContainer in app["containers"]:
assignIpV4(appId, appContainer["name"])
if "port" in appContainer:
if "preferredOutsidePort" in appContainer and "requiresPort" in appContainer and appContainer["requiresPort"]:
validatePort(appContainer["name"], appContainer["preferredOutsidePort"], appId, 2)
elif "preferredOutsidePort" in appContainer:
validatePort(appContainer["name"], appContainer["preferredOutsidePort"], appId, 1)
else:
validatePort(appContainer["name"], appContainer["port"], appId, 0)
else:
# if the container does not define a port, assume 3000, and pass it to the container as env var
validatePort(appContainer["name"], 3000, appId, 0, True)
if "requiredPorts" in appContainer:
for port in appContainer["requiredPorts"]:
validatePort(appContainer["name"], port, appId, 2)
if "requiredUdpPorts" in appContainer:
for port in appContainer["requiredUdpPorts"]:
validatePort(appContainer["name"], port, appId, 2)
def getPortsV4App(app, appId):
implements = ""
if "implements" in app["metadata"]:
implements = app["metadata"]["implements"]
for appContainerName in app["services"].keys():
appContainer = app["services"][appContainerName]
if "enable_networking" in appContainer and not appContainer["enable_networking"]:
return
assignIpV4(appId, appContainerName)
if "port" in appContainer:
validatePort(appContainerName, appContainer["port"], appId, 0, False, implements)
else:
# if the container does not define a port, assume 3000, and pass it to the container as env var
validatePort(appContainerName, 3000, appId, 0, True, implements)
if "required_ports" in appContainer:
if "tcp" in appContainer["required_ports"] and appContainer["required_ports"]["tcp"] != None:
for port in appContainer["required_ports"]["tcp"].keys():
validatePort(appContainerName, port, appId, 2, False, implements)
if "udp" in appContainer["required_ports"] and appContainer["required_ports"]["udp"] != None:
for port in appContainer["required_ports"]["udp"].keys():
validatePort(appContainerName, port, appId, 2, False, implements)

View File

@ -1,98 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
import json
import os
import yaml
scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
nodeRoot = os.path.join(scriptDir, "..")
userFile = os.path.join(nodeRoot, "db", "user.json")
appsDir = os.path.join(nodeRoot, "apps")
with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file:
dependencies = yaml.safe_load(file)
def getUserData():
userData = {}
if os.path.isfile(userFile):
with open(userFile, "r") as f:
userData = json.load(f)
return userData
def getInstalledVirtualApps():
installedApps = []
try:
with open(os.path.join(appsDir, "virtual-apps.json"), "r") as f:
virtual_apps = json.load(f)
userData = getUserData()
for virtual_app in virtual_apps.keys():
for implementation in virtual_apps[virtual_app]:
if "installedApps" in userData and implementation in userData["installedApps"]:
installedApps.append(virtual_app)
except: pass
return installedApps
# Lists all folders in a directory and checks if they are valid
# A folder is valid if it contains an app.yml file
# A folder is invalid if it doesn't contain an app.yml file
def findAndValidateApps(dir: str):
services = ["lnd", "bitcoind"]
userData = getUserData()
if not "installedApps" in userData:
userData["installedApps"] = []
services.extend(userData["installedApps"])
services.extend(getInstalledVirtualApps())
service_str = ",".join(services)
apps = []
for subdir in os.scandir(dir):
if not subdir.is_dir():
continue
app_dir = subdir.path
allowed_app_files = 3
if os.path.isfile(os.path.join(app_dir, "app.yml.jinja")):
allowed_app_files += 1
os.chown(app_dir, 1000, 1000)
os.system(
"docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli preprocess --app-name '{}' /apps/{}/app.yml.jinja /apps/{}/app.yml --services '{}'".format(
dir, dependencies["app-cli"], subdir.name, subdir.name, subdir.name, service_str
)
)
# App should be re-converted considering this may have changed the app.yml
if os.path.isfile(os.path.join(app_dir, "result.yml")):
os.remove(os.path.join(app_dir, "result.yml"))
for subfile in os.scandir(subdir):
if allowed_app_files == 0:
break
if (
subfile.is_file()
and subfile.name.endswith(".jinja")
and subfile.name != "app.yml.jinja"
):
allowed_app_files -= 1
os.chown(app_dir, 1000, 1000)
cmd = "docker run --rm -v {}:/seed -v {}:/.env -v {}:/apps -u 1000:1000 {} /app-cli preprocess-config-file --env-file /.env --app-name '{}' --app-file '/apps/{}/{}' /apps/{}/{} /apps/{}/{} --services '{}' --seed-file /seed".format(
os.path.join(nodeRoot, "db", "citadel-seed", "seed"),
os.path.join(nodeRoot, ".env"),
dir,
dependencies["app-cli"],
subdir.name,
subdir.name,
"app.yml",
subdir.name,
subfile.name,
subdir.name,
subfile.name[:-6],
service_str,
)
print(cmd)
os.system(cmd)
if not os.path.isfile(os.path.join(app_dir, "app.yml")):
print("App {} has no app.yml".format(subdir.name))
else:
apps.append(subdir.name)
return apps

View File

@ -38,7 +38,7 @@ if [ "$1" == "stop" ] || [ "$1" == "start" ] || [ "$1" == "install" ] || [ "$1"
fi
elif [ "$1" == "update" ] && [[ "$2" != "" ]]; then
for app in "${@:2}"; do
"${NODE_ROOT}/app/app-manager.py" --invoked-by-configure update "$app"
"${NODE_ROOT}/app/app-manager.py" update "$app"
done
"${NODE_ROOT}/app/app-manager.py" generate
for app in "${@:2}"; do

5
scripts/configure vendored
View File

@ -312,7 +312,6 @@ def replace_vars(file_path):
return re.sub(r'<(.*?)>', lambda m: get_var(convert_to_upper(m.group(1)), locals(), file_path), file_contents)
templates_to_build = {
"./templates/torrc-empty": ["./tor/torrc-apps", "./tor/torrc-apps-2", "./tor/torrc-apps-3"],
"./templates/torrc-core-sample": "./tor/torrc-core",
"./templates/lnd-sample.conf": "./lnd/lnd.conf",
"./templates/bitcoin-sample.conf": "./bitcoin/bitcoin.conf",
@ -360,10 +359,10 @@ with open("docker-compose.yml", "w") as stream:
if not reconfiguring:
print("Updating apps...\n")
os.system('./scripts/app --invoked-by-configure update')
os.system('./scripts/app update')
else:
print("Updating apps...\n")
os.system('./scripts/app --invoked-by-configure generate')
os.system('./scripts/app generate')
print("Configuring permissions...\n")
os.system('chown -R 1000:1000 {}'.format(CITADEL_ROOT))
# Touch status_dir/configured

View File

@ -1,4 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later