michael.heier/citadel-core
1
0
Fork 1
mirror of https://github.com/runcitadel/core.git synced 2024-12-27 07:05:05 +00:00
Code Issues Projects Releases Wiki Activity

Citadel 0.0.8 (#77)

Browse Source 
Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com>
Co-authored-by: Philipp Walter <philippwalter@pm.me>
This commit is contained in:
Aaron Dewes 2022-09-05 21:01:12 +02:00 committed by GitHub
parent 302774d9f0
commit 3c32a62cc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 550 additions and 1201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

230
app-standard-v1.json
View File

@ -1,230 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Citadel app.yml v1",
"description": "The first draft of Citadel's app.yml format",
"type": "object",
"properties": {
"version": {
"type": [
"string",
"number"
],
"description": "The version of the app.yml format you're using."
},
"metadata": {
"type": "object",
"properties": {
"name": {
"description": "Displayed name of the app",
"type": "string"
},
"version": {
"description": "Displayed version for the app",
"type": "string"
},
"category": {
"description": "The category you'd put the app in",
"type": "string"
},
"tagline": {
"description": "A clever tagline",
"type": "string"
},
"description": {
"description": "A longer description of the app",
"type": "string"
},
"developer": {
"description": "The awesome people behind the app",
"type": "string"
},
"website": {
"description": "Displayed version for the app",
"type": "string"
},
"dependencies": {
"description": "The services the app depends on",
"type": "array",
"items": {
"type": "string"
}
},
"repo": {
"description": "The development repository for your app",
"type": "string"
},
"support": {
"description": "A link to the app support wiki/chat/...",
"type": "string"
},
"gallery": {
"type": "array",
"description": "URLs or paths in the runcitadel/app-images/[app-name] folder with app images",
"items": {
"type": "string"
}
},
"path": {
"description": "The path of the app's visible site the open button should open",
"type": "string"
},
"defaultPassword": {
"description": "The app's default password",
"type": "string"
},
"torOnly": {
"description": "Whether the app is only available over tor",
"type": "boolean"
},
"mainContainer": {
"type": "string",
"description": "The name of the main container for the app. If set, IP, port, and hidden service will be assigned to it automatically."
},
"updateContainer": {
"type": "string",
"description": "The container the developer system should automatically update."
}
},
"required": [
"name",
"version",
"category",
"tagline",
"description",
"developer",
"website",
"repo",
"support",
"gallery"
],
"additionalProperties": false
},
"containers": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"image": {
"type": "string"
},
"permissions": {
"type": "array",
"items": {
"type": "string",
"enum": [
"lnd",
"bitcoind",
"electrum",
"root",
"hw"
]
}
},
"ports": {
"type": "array",
"items": {
"type": [
"string",
"number"
]
}
},
"port": {
"type": "number",
"description": "If this is the main container, the port inside the container which will be exposed to the outside as the port specified in metadata."
},
"environment": {
"type": "object"
},
"data": {
"type": "array",
"description": "An array of at directories in the container the app stores its data in. Can be empty. Please only list top-level directories.",
"items": {
"type": "string"
}
},
"user": {
"type": "string",
"description": "The user the container should run as"
},
"stop_grace_period": {
"type": "string",
"description": "The grace period for stopping the container. Defaults to 1 minute."
},
"depends_on": {
"type": "array",
"description": "The services the container depends on"
},
"entrypoint": {
"type": [
"string",
"array"
],
"description": "The entrypoint for the container"
},
"bitcoin_mount_dir": {
"type": "string",
"description": "Where to mount the bitcoin dir"
},
"command": {
"type": [
"string",
"array"
],
"description": "The command for the container"
},
"init": {
"type": "boolean",
"description": "Whether the container should be run with init"
},
"stop_signal": {
"type": "string",
"description": "The signal to send to the container when stopping"
},
"noNetwork": {
"type": "boolean",
"description": "Set this to true if the container shouldn't get an IP & port exposed."
},
"needsHiddenService": {
"type": "boolean",
"description": "Set this to true if the container should be assigned a hidden service even if it's not the main container."
},
"hiddenServicePort": {
"type": "number",
"description": "Set this to a port if your container exposes multiple ports, but only one should be a hidden service."
},
"hiddenServicePorts": {
"type": "object",
"description": "Set this to a map of service names to hidden service ports if your container exposes multiple ports, and all of them should be hidden services.",
"patternProperties": {
"^[a-zA-Z0-9_]+$": {
"type": [
"number",
"array"
]
}
}
},
"restart": {
"type": "string",
"description": "When the container should restart. Can be 'always' or 'on-failure'."
}
},
"additionalProperties": false,
"required": [
"name",
"image"
]
},
"additionalProperties": false
}
},
"required": [
"metadata",
"containers"
],
"additionalProperties": false
}

3
app-standard-v1.json.license
View File

@ -1,3 +0,0 @@
SPDX-FileCopyrightText: 2021 Citadel and contributors
SPDX-License-Identifier: GPL-3.0-or-later

4
app-system/sources.list
View File

@ -3,9 +3,9 @@
# SPDX-License-Identifier: GPL-3.0-or-later
# A collection of fully FLOSS app definitions and FLOSS apps for Citadel.
https://github.com/runcitadel/apps v3-stable
https://github.com/runcitadel/apps v4-stable
# Some apps modified version of Umbrel apps, and their app definitions aren't FLOSS yet.
# Include them anyway, but as a separate repo.
# Add a # to the line below to disable the repo and only use FLOSS apps.
https://github.com/runcitadel/apps-nonfree v3-stable
https://github.com/runcitadel/apps-nonfree v4-stable

194
app/app-standard-v1.yml
View File

@ -1,194 +0,0 @@
# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema
$schema: https://json-schema.org/draft/2020-12/schema
title: Citadel app.yml v1
description: The first draft of Citadel's app.yml format
type: object
properties:
version:
type:
- string
- number
description: The version of the app.yml format you're using.
metadata:
type: object
properties:
name:
description: Displayed name of the app
type: string
version:
description: Displayed version for the app
type: string
category:
description: The category you'd put the app in
type: string
tagline:
description: A clever tagline
type: string
description:
description: A longer description of the app
type: string
developer:
description: The awesome people behind the app
type: string
website:
description: Displayed version for the app
type: string
dependencies:
description: The services the app depends on
type: array
items:
type: string
repo:
description: The development repository for your app
type: string
support:
description: A link to the app support wiki/chat/...
type: string
gallery:
type: array
description: >-
URLs or paths in the runcitadel/app-images/[app-name] folder with app
images
items:
type: string
path:
description: The path of the app's visible site the open button should open
type: string
defaultPassword:
description: The app's default password
type: string
torOnly:
description: Whether the app is only available over tor
type: boolean
mainContainer:
type: string
description: >-
The name of the main container for the app. If set, IP, port, and
hidden service will be assigned to it automatically.
updateContainer:
type: string
description: The container the developer system should automatically update.
required:
- name
- version
- category
- tagline
- description
- developer
- website
- repo
- support
- gallery
additionalProperties: false
containers:
type: array
items:
type: object
properties:
name:
type: string
image:
type: string
permissions:
type: array
items:
type: string
enum:
- lnd
- bitcoind
- electrum
- root
- hw
ports:
type: array
items:
type:
- string
- number
port:
type: number
description: >-
If this is the main container, the port inside the container which
will be exposed to the outside as the port specified in metadata.
environment:
type: object
data:
type: array
description: >-
An array of at directories in the container the app stores its data
in. Can be empty. Please only list top-level directories.
items:
type: string
user:
type: string
description: The user the container should run as
stop_grace_period:
type: string
description: The grace period for stopping the container. Defaults to 1 minute.
depends_on:
type: array
description: The services the container depends on
entrypoint:
type:
- string
- array
description: The entrypoint for the container
bitcoin_mount_dir:
type: string
description: Where to mount the bitcoin dir
command:
type:
- string
- array
description: The command for the container
init:
type: boolean
description: Whether the container should be run with init
stop_signal:
type: string
description: The signal to send to the container when stopping
noNetwork:
type: boolean
description: >-
Set this to true if the container shouldn't get an IP & port
exposed.
needsHiddenService:
type: boolean
description: >-
Set this to true if the container should be assigned a hidden
service even if it's not the main container.
hiddenServicePort:
type: number
description: >-
Set this to a port if your container exposes multiple ports, but
only one should be a hidden service.
hiddenServicePorts:
type: object
description: >-
Set this to a map of service names to hidden service ports if your
container exposes multiple ports, and all of them should be hidden
services.
patternProperties:
^[a-zA-Z0-9_]+$:
type:
- number
- array
restart:
type: string
description: When the container should restart. Can be 'always' or 'on-failure'.
additionalProperties: false
required:
- name
- image
additionalProperties: false
required:
- metadata
- containers
additionalProperties: false

49
app/lib/citadelutils.py
View File

@ -3,6 +3,8 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import re
import fcntl
import os
# Helper functions
@ -44,29 +46,6 @@ def parse_dotenv(file_path):
exit(1)
return envVars
# Combines an object and a class
# If the key exists in both objects, the value of the second object is used
# If the key does not exist in the first object, the value from the second object is used
# If a key contains a list, the second object's list is appended to the first object's list
# If a key contains another object, these objects are combined
def combineObjectAndClass(theClass, obj: dict):
for key, value in obj.items():
if key in theClass.__dict__:
if isinstance(value, list):
if isinstance(theClass.__dict__[key], list):
theClass.__dict__[key].extend(value)
else:
theClass.__dict__[key] = [theClass.__dict__[key]] + value
elif isinstance(value, dict):
if isinstance(theClass.__dict__[key], dict):
theClass.__dict__[key].update(value)
else:
theClass.__dict__[key] = {theClass.__dict__[key]: value}
else:
theClass.__dict__[key] = value
else:
theClass.__dict__[key] = value
def is_builtin_type(obj):
return isinstance(obj, (int, float, str, bool, list, dict))
@ -100,3 +79,27 @@ def classToDict(theClass):
obj[key] = classToDict(value)
return obj
class FileLock:
"""Implements a file-based lock using flock(2).
The lock file is saved in directory dir with name lock_name.
dir is the current directory by default.
"""
def __init__(self, lock_name, dir="."):
self.lock_file = open(os.path.join(dir, lock_name), "w")
def acquire(self, blocking=True):
"""Acquire the lock.
If the lock is not already acquired, return None. If the lock is
acquired and blocking is True, block until the lock is released. If
the lock is acquired and blocking is False, raise an IOError.
"""
ops = fcntl.LOCK_EX
if not blocking:
ops |= fcntl.LOCK_NB
fcntl.flock(self.lock_file, ops)
def release(self):
"""Release the lock. Return None even if lock not currently acquired"""
fcntl.flock(self.lock_file, fcntl.LOCK_UN)

24
app/lib/composegenerator/next/stage1.py
View File

@ -1,24 +0,0 @@
from lib.citadelutils import classToDict
from lib.composegenerator.shared.env import validateEnv
from lib.composegenerator.v3.types import App, generateApp
from lib.composegenerator.v3.generate import convertContainerPermissions
def createCleanConfigFromV3(app: dict, nodeRoot: str):
parsedApp: App = generateApp(app)
for container in range(len(parsedApp.containers)):
# TODO: Make this dynamic and not hardcoded
if parsedApp.containers[container].requires and "c-lightning" in parsedApp.containers[container].requires:
parsedApp.containers[container] = None
parsedApp = convertContainerPermissions(parsedApp)
parsedApp = validateEnv(parsedApp)
finalApp = classToDict(parsedApp)
try:
finalApp['permissions'] = finalApp['metadata']['dependencies']
except:
finalApp['permissions'] = []
finalApp['id'] = finalApp['metadata']['id']
del finalApp['metadata']
# Set version of the cache file format
finalApp['version'] = "1"
return finalApp

2
app/lib/composegenerator/shared/env.py
View File

@ -4,7 +4,7 @@
import re
from typing import Union
from lib.composegenerator.v1.types import App
from lib.composegenerator.v2.types import App
from lib.composegenerator.shared.const import always_allowed_env
from lib.citadelutils import checkArrayContainsAllElements, getEnvVars

27
app/lib/composegenerator/shared/main.py
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
# Main functions
from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container
from lib.composegenerator.v2.types import App, AppStage3
from lib.composegenerator.shared.const import permissions
@ -27,28 +27,3 @@ def convertContainersToServices(app: AppStage3) -> AppStage3:
del app.containers
app.services = services
return app
# Converts the data of every container in app.containers to a volume, which is then added to the app
def convertDataDirToVolume(app: App) -> AppStage2:
for container in app.containers:
# Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
# Also, a datadir shouldn't start with a /
for dataDir in container.data:
if dataDir.find("..") == -1 and dataDir[0] != "/":
container.volumes.append(
'${APP_DATA_DIR}/' + dataDir)
else:
print("Data dir " + dataDir +
" contains invalid characters")
del container.data
if container.bitcoin_mount_dir != None:
if not 'bitcoind' in container.permissions:
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
# Skip this container
continue
# Also skip the container if container.bitcoin_mount_dir contains a :
if container.bitcoin_mount_dir.find(":") == -1:
container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir)
del container.bitcoin_mount_dir
return app

171
app/lib/composegenerator/shared/networking.py Normal file
View File

@ -0,0 +1,171 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
import json
from os import path
import random
from lib.composegenerator.shared.utils.networking import getContainerHiddenService
from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container
from lib.citadelutils import parse_dotenv
from dacite import from_dict
def getMainContainer(app: App) -> Container:
if len(app.containers) == 1:
return app.containers[0]
else:
for container in app.containers:
# Main is recommended, support web for easier porting from Umbrel
if container.name == 'main' or container.name == 'web':
return container
# Fallback to first container
return app.containers[0]
def assignIpV4(appId: str, containerName: str):
scriptDir = path.dirname(path.realpath(__file__))
nodeRoot = path.join(scriptDir, "..", "..", "..", "..")
networkingFile = path.join(nodeRoot, "apps", "networking.json")
envFile = path.join(nodeRoot, ".env")
cleanContainerName = containerName.strip()
# If the name still contains a newline, throw an error
if cleanContainerName.find("\n") != -1:
raise Exception("Newline in container name")
env_var = "APP_{}_{}_IP".format(
appId.upper().replace("-", "_"),
cleanContainerName.upper().replace("-", "_")
)
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
# can be used
usedIps = []
networkingData = {}
if path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ip_addresses' in networkingData:
usedIps = list(networkingData['ip_addresses'].values())
else:
networkingData['ip_addresses'] = {}
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
# If the IP is already in use, it will be tried again until it's not in use
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
# Throw an error, because no more IPs can be used
if len(usedIps) == 235:
raise Exception("No more IPs can be used")
if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']:
ip = networkingData['ip_addresses']["{}-{}".format(
appId, cleanContainerName)]
else:
while True:
ip = "10.21.21." + str(random.randint(20, 255))
if ip not in usedIps:
networkingData['ip_addresses']["{}-{}".format(
appId, cleanContainerName)] = ip
break
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
return
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, ip))
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
# Strip leading/trailing whitespace from container.name
container.name = container.name.strip()
# If the name still contains a newline, throw an error
if container.name.find("\n") != -1:
raise Exception("Newline in container name")
env_var = "APP_{}_{}_IP".format(
appId.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
# can be used
usedIps = []
networkingData = {}
if path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ip_addresses' in networkingData:
usedIps = list(networkingData['ip_addresses'].values())
else:
networkingData['ip_addresses'] = {}
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
# If the IP is already in use, it will be tried again until it's not in use
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
# Throw an error, because no more IPs can be used
if len(usedIps) == 235:
raise Exception("No more IPs can be used")
if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
ip = networkingData['ip_addresses']["{}-{}".format(
appId, container.name)]
else:
while True:
ip = "10.21.21." + str(random.randint(20, 255))
if ip not in usedIps:
networkingData['ip_addresses']["{}-{}".format(
appId, container.name)] = ip
break
container.networks = from_dict(data_class=NetworkConfig, data={'default': {
'ipv4_address': "$" + env_var}})
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
return container
# Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, ip))
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
return container
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
for container in app.containers:
if container.network_mode and container.network_mode == "host":
continue
if container.noNetwork:
# Check if port is defined for the container
if container.port:
raise Exception("Port defined for container without network")
if getMainContainer(app).name == container.name:
raise Exception("Main container without network")
# Skip this iteration of the loop
continue
container = assignIp(container, app.metadata.id,
networkingFile, envFile)
return app
def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3:
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
hiddenServices = ""
mainContainer = getMainContainer(app)
for container in app.containers:
if container.network_mode and container.network_mode == "host":
continue
env_var = "APP_{}_{}_IP".format(
app.metadata.id.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
hiddenServices += getContainerHiddenService(
app.metadata, container, dotEnv[env_var], container.name == mainContainer.name)
if container.hiddenServicePorts:
del container.hiddenServicePorts
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(hiddenServices)
return app

0
app/lib/composegenerator/v2/utils/networking.py → app/lib/composegenerator/shared/utils/networking.py
View File

30
app/lib/composegenerator/v1/generate.py
View File

@ -1,30 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
from lib.composegenerator.v1.types import App, AppStage4, generateApp
from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
from lib.composegenerator.shared.env import validateEnv
from lib.citadelutils import classToDict
import os
def createComposeConfigFromV1(app: dict, nodeRoot: str):
envFile = os.path.join(nodeRoot, ".env")
networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
newApp: App = generateApp(app)
newApp = convertContainerPermissions(newApp)
validateEnv(newApp)
newApp = convertDataDirToVolume(newApp)
newApp = configureIps(newApp, networkingFile, envFile)
newApp = configureMainPort(newApp, nodeRoot)
configureHiddenServices(newApp, nodeRoot)
finalConfig: AppStage4 = convertContainersToServices(newApp)
newApp = classToDict(finalConfig)
del newApp['metadata']
if "version" in newApp:
del newApp["version"]
# Set version to 3.8 (current compose file version)
newApp = {'version': '3.8', **newApp}
return newApp

226
app/lib/composegenerator/v1/networking.py
View File

@ -1,226 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
from dacite import from_dict
from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig
from lib.citadelutils import parse_dotenv
import json
from os import path
import random
from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService
def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
# Strip leading/trailing whitespace from container.name
container.name = container.name.strip()
# If the name still contains a newline, throw an error
if container.name.find("\n") != -1:
raise Exception("Newline in container name")
env_var = "APP_{}_{}_IP".format(
appId.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
# can be used
usedIps = []
networkingData = {}
if path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ip_addresses' in networkingData:
usedIps = list(networkingData['ip_addresses'].values())
else:
networkingData['ip_addresses'] = {}
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
# If the IP is already in use, it will be tried again until it's not in use
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
# Throw an error, because no more IPs can be used
if len(usedIps) == 235:
raise Exception("No more IPs can be used")
if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
ip = networkingData['ip_addresses']["{}-{}".format(
appId, container.name)]
else:
while True:
ip = "10.21.21." + str(random.randint(20, 255))
if ip not in usedIps:
networkingData['ip_addresses']["{}-{}".format(
appId, container.name)] = ip
break
container.networks = from_dict(data_class=NetworkConfig, data={'default': {
'ipv4_address': "$" + env_var}})
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
return container
# Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, ip))
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
return container
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
# Strip leading/trailing whitespace from container.name
container.name = container.name.strip()
# If the name still contains a newline, throw an error
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
raise Exception("Newline or space in container name")
env_var = "APP_{}_{}_PORT".format(
appId.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
port = getFreePort(networkingFile, appId)
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
return {"port": port, "env_var": "${{{}}}".format(env_var)}
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, port))
# This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
# where the outer {{ }} will be replaced by {} in the returned string
return {"port": port, "env_var": "${{{}}}".format(env_var)}
def getMainContainer(app: dict):
if len(app.containers) == 1:
return app.containers[0]
else:
if not app.metadata.mainContainer:
app.metadata.mainContainer = 'main'
for container in app.containers:
if container.name == app.metadata.mainContainer:
return container
raise Exception(
"No main container found for app {}".format(app.metadata.name))
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
registryFile = path.join(nodeRoot, "apps", "registry.json")
registry: list = []
if path.isfile(registryFile):
with open(registryFile, 'r') as f:
registry = json.load(f)
else:
raise Exception("Registry file not found")
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
mainContainer = getMainContainer(app)
portDetails = assignPort(mainContainer, app.metadata.id, path.join(
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
containerPort = portDetails['port']
portAsEnvVar = portDetails['env_var']
portToAppend = portAsEnvVar
mainPort = False
if mainContainer.port:
portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port)
mainPort = mainContainer.port
del mainContainer.port
else:
portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar)
if mainContainer.ports:
mainContainer.ports.append(portToAppend)
# Set the main port to the first port in the list, if it contains a :, it's the port after the :
# If it doesn't contain a :, it's the port itself
if mainPort == False:
mainPort = mainContainer.ports[0]
if mainPort.find(":") != -1:
mainPort = mainPort.split(":")[1]
else:
mainContainer.ports = [portToAppend]
if mainPort == False:
mainPort = portDetails['port']
mainContainer = assignIp(mainContainer, app.metadata.id, path.join(
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
# If the IP wasn't in dotenv before, now it should be
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace(
"-", "_"), mainContainer.name.upper().replace("-", "_"))]
hiddenservice = getHiddenService(
app.metadata.name, app.metadata.id, containerIP, mainPort)
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(hiddenservice)
# Also set the port in metadata
app.metadata.port = int(containerPort)
for registryApp in registry:
if registryApp['id'] == app.metadata.id:
registry[registry.index(registryApp)]['port'] = int(containerPort)
break
with open(registryFile, 'w') as f:
json.dump(registry, f, indent=4, sort_keys=True)
return app
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
for container in app.containers:
if container.noNetwork:
# Check if port is defined for the container
if container.port:
raise Exception("Port defined for container without network")
if app.metadata.mainContainer == container.name:
raise Exception("Main container without network")
# Skip this iteration of the loop
continue
container = assignIp(container, app.metadata.id,
networkingFile, envFile)
return app
def configureHiddenServices(app: dict, nodeRoot: str) -> None:
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
hiddenServices = ""
if len(app.containers) == 1:
mainContainer = app.containers[0]
else:
mainContainer = None
if app.metadata.mainContainer == None:
app.metadata.mainContainer = 'main'
for container in app.containers:
if container.name == app.metadata.mainContainer:
mainContainer = container
break
if mainContainer is None:
raise Exception("No main container found")
for container in app.containers:
env_var = "APP_{}_{}_IP".format(
app.metadata.id.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
hiddenServices += getContainerHiddenService(
app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name)
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(hiddenServices)

151
app/lib/composegenerator/v1/types.py
View File

@ -1,151 +0,0 @@
from typing import Union, List
from dataclasses import dataclass, field
from dacite import from_dict
@dataclass
class Metadata:
id: str
name: str
version: str
category: str
tagline: str
description: str
developer: str
website: str
repo: str
support: str
gallery: List[str] = field(default_factory=list)
dependencies: List[str] = field(default_factory=list)
mainContainer: Union[str, None] = None
updateContainer: Union[str, None] = None
path: str = ""
defaultPassword: str = ""
torOnly: bool = False
@dataclass
class Container:
name: str
image: str
permissions: list = field(default_factory=list)
ports: list = field(default_factory=list)
port: Union[int, None] = None
environment: Union[dict, None] = None
data: list = field(default_factory=list)
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: list = field(default_factory=list)
entrypoint: Union[List[str], str] = field(default_factory=list)
bitcoin_mount_dir: Union[str, None] = None
command: Union[List[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
environment_allow: list = field(default_factory=list)
# Only added later
volumes: list = field(default_factory=list)
restart: Union[str, None] = None
@dataclass
class App:
version: Union[str, int]
metadata: Metadata
containers: List[Container]
# Generate an app instance from an app dict
def generateApp(appDict):
return from_dict(data_class=App, data=appDict)
@dataclass
class Network:
ipv4_address: Union[str, None] = None
@dataclass
class NetworkConfig:
default: Network
# After converting data dir and defining volumes, stage 2
@dataclass
class ContainerStage2:
id: str
name: str
image: str
permissions: List[str] = field(default_factory=list)
ports: list = field(default_factory=list)
environment: Union[dict, None] = None
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: List[str] = field(default_factory=list)
entrypoint: Union[List[str], str] = field(default_factory=list)
command: Union[List[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
volumes: List[str] = field(default_factory=list)
networks: NetworkConfig = field(default_factory=NetworkConfig)
restart: Union[str, None] = None
@dataclass
class AppStage2:
version: Union[str, int]
metadata: Metadata
containers: List[ContainerStage2]
@dataclass
class MetadataStage3:
id: str
name: str
version: str
category: str
tagline: str
description: str
developer: str
website: str
dependencies: List[str]
repo: str
support: str
gallery: List[str]
mainContainer: Union[str, None] = None
updateContainer: Union[str, None] = None
path: str = ""
defaultPassword: str = ""
torOnly: bool = False
@dataclass
class AppStage3:
version: Union[str, int]
metadata: MetadataStage3
containers: List[ContainerStage2]
@dataclass
class ContainerStage4:
id: str
name: str
image: str
ports: list = field(default_factory=list)
environment: Union[dict, None] = None
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: List[str] = field(default_factory=list)
entrypoint: Union[List[str], str] = field(default_factory=list)
command: Union[List[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
volumes: List[str] = field(default_factory=list)
networks: NetworkConfig = field(default_factory=NetworkConfig)
restart: Union[str, None] = None
@dataclass
class AppStage4:
version: Union[str, int]
metadata: MetadataStage3
services: List[ContainerStage4]

118
app/lib/composegenerator/v1/utils/networking.py
View File

@ -1,118 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
import json
import os
import random
from lib.composegenerator.v1.types import Container
def getFreePort(networkingFile: str, appId: str):
# Ports used currently in Citadel
usedPorts = [
# Dashboard
80,
# Sometimes used by nginx with some setups
433,
# Dashboard SSL
443,
# Bitcoin Core P2P
8333,
# LND gRPC
10009,
# LND REST
8080,
# Electrum Server
50001,
# Tor Proxy
9050,
]
networkingData = {}
if os.path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ports' in networkingData:
usedPorts += list(networkingData['ports'].values())
else:
networkingData['ports'] = {}
if appId in networkingData['ports']:
return networkingData['ports'][appId]
while True:
port = str(random.randint(1024, 49151))
if port not in usedPorts:
# Check if anyhing is listening on the specific port
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
networkingData['ports'][appId] = port
break
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
return port
def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str:
hiddenServices = '''
# {} Hidden Service
HiddenServiceDir /var/lib/tor/app-{}
'''.format(name, id)
for port in ports:
hiddenServices += 'HiddenServicePort {} {}:{}'.format(
port, internalIp, port)
hiddenServices += "\n"
return hiddenServices
def getHiddenServiceString(name: str, id: str, internalPort, internalIp: str, publicPort) -> str:
return '''
# {} Hidden Service
HiddenServiceDir /var/lib/tor/app-{}
HiddenServicePort {} {}:{}
'''.format(name, id, publicPort, internalIp, internalPort)
def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str:
return getHiddenServiceString(appName, appId, appPort, appIp, "80")
def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str:
if not container.needsHiddenService and not isMainContainer:
return ""
if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer:
print("Container {} for app {} isn't compatible with hidden service assignment".format(
container.name, appName))
return ""
if isMainContainer:
if not container.hiddenServicePorts:
return ""
# hiddenServicePorts is a map of hidden service name to port
# We need to generate a hidden service for each one
hiddenServices = ""
for name, port in container.hiddenServicePorts.items():
if ".." in name:
print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName))
# If port is a list, use getHiddenServiceMultiPort
if isinstance(port, list):
hiddenServices += getHiddenServiceMultiPort("{} {}".format(appName, name), "{}-{}".format(
appId, name), containerIp, port)
else:
hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format(
appId, name), port, containerIp, port)
del container.hiddenServicePorts
return hiddenServices
del container.needsHiddenService
if not container.port:
data = getHiddenServiceString(appName + container.name, "{}-{}".format(
appId, container.name), container.hiddenServicePort, containerIp, "80")
del container.hiddenServicePort
return data
else:
return getHiddenServiceString(appName + container.name, "{}-{}".format(
appId, container.name), container.port, containerIp, container.port)

25
app/lib/composegenerator/v2/generate.py
View File

@ -3,15 +3,34 @@
# SPDX-License-Identifier: GPL-3.0-or-later
from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp
from lib.composegenerator.v2.networking import configureHiddenServices, configureIps, configureMainPort
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
from lib.composegenerator.v2.networking import configureMainPort
from lib.composegenerator.shared.networking import configureHiddenServices, configureIps
from lib.composegenerator.shared.main import convertContainerPermissions, convertContainersToServices
from lib.composegenerator.shared.env import validateEnv
from lib.citadelutils import classToDict
import os
def convertDataDirToVolumeGen2(app: App) -> AppStage2:
app = convertDataDirToVolume(app)
for container in app.containers:
# Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
# Also, a datadir shouldn't start with a /
for dataDir in container.data:
if dataDir.find("..") == -1 and dataDir[0] != "/":
container.volumes.append(
'${APP_DATA_DIR}/' + dataDir)
else:
print("Data dir " + dataDir +
" contains invalid characters")
del container.data
if container.bitcoin_mount_dir != None:
if not 'bitcoind' in container.permissions:
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
# Skip this container
continue
# Also skip the container if container.bitcoin_mount_dir contains a :
if container.bitcoin_mount_dir.find(":") == -1:
container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir)
del container.bitcoin_mount_dir
if container.lnd_mount_dir != None:
if not 'lnd' in container.permissions:
print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name))

124
app/lib/composegenerator/v2/networking.py
View File

@ -2,14 +2,59 @@
#
# SPDX-License-Identifier: GPL-3.0-or-later
from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container
from lib.citadelutils import parse_dotenv
from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container
import json
from os import path
import os
import random
from lib.composegenerator.v2.utils.networking import getContainerHiddenService
from lib.composegenerator.v1.networking import assignIp, assignPort
from lib.composegenerator.shared.networking import assignIp
from lib.citadelutils import FileLock
def getFreePort(networkingFile: str, appId: str):
# Ports used currently in Citadel
usedPorts = [
# Dashboard
80,
# Sometimes used by nginx with some setups
433,
# Dashboard SSL
443,
# Bitcoin Core P2P
8333,
# LND gRPC
10009,
# LND REST
8080,
# Electrum Server
50001,
# Tor Proxy
9050,
]
networkingData = {}
if path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ports' in networkingData:
usedPorts += list(networkingData['ports'].values())
else:
networkingData['ports'] = {}
if appId in networkingData['ports']:
return networkingData['ports'][appId]
while True:
port = str(random.randint(1024, 49151))
if port not in usedPorts:
# Check if anyhing is listening on the specific port
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
networkingData['ports'][appId] = port
break
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
return port
def getMainContainer(app: App) -> Container:
if len(app.containers) == 1:
@ -22,8 +67,36 @@ def getMainContainer(app: App) -> Container:
# Fallback to first container
return app.containers[0]
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
# Strip leading/trailing whitespace from container.name
container.name = container.name.strip()
# If the name still contains a newline, throw an error
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
raise Exception("Newline or space in container name")
env_var = "APP_{}_{}_PORT".format(
appId.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
port = getFreePort(networkingFile, appId)
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
return {"port": port, "env_var": "${{{}}}".format(env_var)}
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, port))
# This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
# where the outer {{ }} will be replaced by {} in the returned string
return {"port": port, "env_var": "${{{}}}".format(env_var)}
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
lock = FileLock("citadel_registry_lock", dir="/tmp")
lock.acquire()
registryFile = path.join(nodeRoot, "apps", "registry.json")
registry: list = []
if path.isfile(registryFile):
@ -81,48 +154,5 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
with open(registryFile, 'w') as f:
json.dump(registry, f, indent=4, sort_keys=True)
return app
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
for container in app.containers:
if container.network_mode and container.network_mode == "host":
continue
if container.noNetwork:
# Check if port is defined for the container
if container.port:
raise Exception("Port defined for container without network")
if getMainContainer(app).name == container.name:
raise Exception("Main container without network")
# Skip this iteration of the loop
continue
container = assignIp(container, app.metadata.id,
networkingFile, envFile)
return app
def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3:
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
hiddenServices = ""
mainContainer = getMainContainer(app)
for container in app.containers:
if container.network_mode and container.network_mode == "host":
continue
env_var = "APP_{}_{}_IP".format(
app.metadata.id.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
hiddenServices += getContainerHiddenService(
app.metadata, container, dotEnv[env_var], container.name == mainContainer.name)
if container.hiddenServicePorts:
del container.hiddenServicePorts
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(hiddenServices)
lock.release()
return app

4
app/lib/composegenerator/v3/generate.py
View File

@ -5,9 +5,9 @@
import os
from lib.citadelutils import classToDict
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainersToServices
from lib.composegenerator.shared.main import convertContainersToServices
from lib.composegenerator.shared.env import validateEnv
from lib.composegenerator.v2.networking import configureIps, configureHiddenServices
from lib.composegenerator.shared.networking import configureIps, configureHiddenServices
from lib.composegenerator.v3.types import App, AppStage2, AppStage4, generateApp
from lib.composegenerator.v3.networking import configureMainPort

12
app/lib/composegenerator/v3/networking.py
View File

@ -2,13 +2,11 @@
#
# SPDX-License-Identifier: GPL-3.0-or-later
from lib.composegenerator.v3.types import App, AppStage2, AppStage3, Container
from lib.citadelutils import parse_dotenv
from lib.composegenerator.v3.types import App, AppStage2, AppStage3
import json
from os import path
import random
from lib.composegenerator.v1.networking import assignIp, assignPort
from lib.composegenerator.shared.networking import assignIp
from lib.citadelutils import FileLock
def getMainContainerIndex(app: App):
if len(app.containers) == 1:
@ -27,6 +25,8 @@ def getMainContainerIndex(app: App):
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
lock = FileLock("citadel_registry_lock", dir="/tmp")
lock.acquire()
registryFile = path.join(nodeRoot, "apps", "registry.json")
portsFile = path.join(nodeRoot, "apps", "ports.json")
envFile = path.join(nodeRoot, ".env")
@ -101,8 +101,8 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
with open(registryFile, 'w') as f:
json.dump(registry, f, indent=4, sort_keys=True)
lock.release()
with open(envFile, 'a') as f:
f.write("{}={}\n".format(portAsEnvVar, app.metadata.port))
return app

92
app/lib/manage.py
View File

@ -6,6 +6,7 @@ import stat
import sys
import tempfile
import threading
import random
from typing import List
from sys import argv
import os
@ -25,16 +26,14 @@ except Exception:
print("Continuing anyway, but some features won't be available,")
print("for example checking for app updates")
from lib.composegenerator.v1.generate import createComposeConfigFromV1
from lib.composegenerator.v2.generate import createComposeConfigFromV2
from lib.composegenerator.v3.generate import createComposeConfigFromV3
from lib.validate import findAndValidateApps
from lib.metadata import getAppRegistry
from lib.entropy import deriveEntropy
from lib.citadelutils import FileLock
# For an array of threads, join them and wait for them to finish
def joinThreads(threads: List[threading.Thread]):
for thread in threads:
thread.join()
@ -50,26 +49,58 @@ updateIgnore = os.path.join(appsDir, ".updateignore")
appDataDir = os.path.join(nodeRoot, "app-data")
userFile = os.path.join(nodeRoot, "db", "user.json")
legacyScript = os.path.join(nodeRoot, "scripts", "app")
with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file:
dependencies = yaml.safe_load(file)
# Returns a list of every argument after the second one in sys.argv joined into a string by spaces
def getArguments():
arguments = ""
for i in range(3, len(argv)):
arguments += argv[i] + " "
return arguments
def handleAppV4(app):
composeFile = os.path.join(appsDir, app, "docker-compose.yml")
os.chown(os.path.join(appsDir, app), 1000, 1000)
os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli convert --app-name '{}' --port-map /apps/ports.json /apps/{}/app.yml /apps/{}/result.yml".format(appsDir, dependencies['app-cli'], app, app, app))
with open(os.path.join(appsDir, app, "result.yml"), "r") as resultFile:
resultYml = yaml.safe_load(resultFile)
with open(composeFile, "w") as dockerComposeFile:
yaml.dump(resultYml["spec"], dockerComposeFile)
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(os.path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(resultYml["new_tor_entries"])
mainPort = resultYml["port"]
registryFile = os.path.join(nodeRoot, "apps", "registry.json")
registry: list = []
lock = FileLock("citadel_registry_lock", dir="/tmp")
lock.acquire()
if os.path.isfile(registryFile):
with open(registryFile, 'r') as f:
registry = json.load(f)
else:
raise Exception("Registry file not found")
for registryApp in registry:
if registryApp['id'] == app:
registry[registry.index(registryApp)]['port'] = mainPort
break
with open(registryFile, 'w') as f:
json.dump(registry, f, indent=4, sort_keys=True)
lock.release()
def getAppYml(name):
with open(os.path.join(appsDir, "sourceMap.json"), "r") as f:
sourceMap = json.load(f)
if not name in sourceMap:
print("Warning: App {} is not in the source map".format(name))
print("Warning: App {} is not in the source map".format(name), file=sys.stderr)
sourceMap = {
name: {
"githubRepo": "runcitadel/core",
"branch": "v2"
"githubRepo": "runcitadel/apps",
"branch": "v4-stable"
}
}
url = 'https://raw.githubusercontent.com/{}/{}/apps/{}/app.yml'.format(sourceMap[name]["githubRepo"], sourceMap[name]["branch"], name)
@ -81,24 +112,48 @@ def getAppYml(name):
def update(verbose: bool = False):
apps = findAndValidateApps(appsDir)
portCache = {}
try:
with open(os.path.join(appsDir, "ports.cache.json"), "w") as f:
portCache = json.load(f)
except Exception: pass
# The compose generation process updates the registry, so we need to get it set up with the basics before that
registry = getAppRegistry(apps, appsDir)
registry = getAppRegistry(apps, appsDir, portCache)
with open(os.path.join(appsDir, "registry.json"), "w") as f:
json.dump(registry["metadata"], f, sort_keys=True)
with open(os.path.join(appsDir, "ports.json"), "w") as f:
json.dump(registry["ports"], f, sort_keys=True)
with open(os.path.join(appsDir, "ports.cache.json"), "w") as f:
json.dump(registry["portCache"], f, sort_keys=True)
with open(os.path.join(appsDir, "virtual-apps.json"), "w") as f:
json.dump(registry["virtual_apps"], f, sort_keys=True)
print("Wrote registry to registry.json")
os.system("docker pull {}".format(dependencies['app-cli']))
threads = list()
# Loop through the apps and generate valid compose files from them, then put these into the app dir
for app in apps:
try:
composeFile = os.path.join(appsDir, app, "docker-compose.yml")
appYml = os.path.join(appsDir, app, "app.yml")
with open(appYml, 'r') as f:
appDefinition = yaml.safe_load(f)
if 'citadel_version' in appDefinition:
thread = threading.Thread(target=handleAppV4, args=(app,))
thread.start()
threads.append(thread)
else:
appCompose = getApp(appDefinition, app)
with open(composeFile, "w") as f:
appCompose = getApp(appYml, app)
if appCompose:
f.write(yaml.dump(appCompose, sort_keys=False))
if verbose:
print("Wrote " + app + " to " + composeFile)
except Exception as err:
print("Failed to convert app {}".format(app))
print(err)
joinThreads(threads)
print("Generated configuration successfully")
@ -158,23 +213,16 @@ def stopInstalled():
joinThreads(threads)
# Loads an app.yml and converts it to a docker-compose.yml
def getApp(appFile: str, appId: str):
with open(appFile, 'r') as f:
app = yaml.safe_load(f)
def getApp(app, appId: str):
if not "metadata" in app:
raise Exception("Error: Could not find metadata in " + appFile)
raise Exception("Error: Could not find metadata in " + appId)
app["metadata"]["id"] = appId
if 'version' in app and str(app['version']) == "1":
print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId))
return createComposeConfigFromV1(app, nodeRoot)
elif 'version' in app and str(app['version']) == "2":
print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId))
if 'version' in app and str(app['version']) == "2":
print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId))
return createComposeConfigFromV2(app, nodeRoot)
elif 'version' in app and str(app['version']) == "3":
print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId))
return createComposeConfigFromV3(app, nodeRoot)
else:
raise Exception("Error: Unsupported version of app.yml")

91
app/lib/metadata.py
View File

@ -4,14 +4,10 @@
import os
import yaml
import traceback
from lib.composegenerator.next.stage1 import createCleanConfigFromV3
from lib.composegenerator.v2.networking import getMainContainer
from lib.composegenerator.v1.networking import getFreePort
from lib.composegenerator.shared.networking import assignIpV4
from lib.entropy import deriveEntropy
from typing import List
import json
import random
appPorts = {}
appPortMap = {}
@ -37,15 +33,21 @@ def appPortsToMap():
# Also check the path and defaultPassword and set them to an empty string if they don't exist
# In addition, set id on the metadata to the name of the app
# Return a list of all app's metadata
def getAppRegistry(apps, app_path):
def getAppRegistry(apps, app_path, portCache):
app_metadata = []
virtual_apps = {}
appPorts = portCache
for app in apps:
app_yml_path = os.path.join(app_path, app, 'app.yml')
app_cache_path = os.path.join(app_path, app, 'app.cache.json')
if os.path.isfile(app_yml_path):
try:
with open(app_yml_path, 'r') as f:
app_yml = yaml.safe_load(f.read())
version = False
if 'version' in app_yml:
version = int(app_yml['version'])
elif 'citadel_version' in app_yml:
version = int(app_yml['citadel_version'])
metadata: dict = app_yml['metadata']
metadata['id'] = app
metadata['path'] = metadata.get('path', '')
@ -54,20 +56,27 @@ def getAppRegistry(apps, app_path):
metadata['defaultPassword'] = deriveEntropy("app-{}-seed".format(app))
if "mainContainer" in metadata:
metadata.pop("mainContainer")
if "implements" in metadata:
implements = metadata["implements"]
if implements not in virtual_apps:
virtual_apps[implements] = []
virtual_apps[implements].append(app)
app_metadata.append(metadata)
if(app_yml["version"] != 3):
if version < 3:
getPortsOldApp(app_yml, app)
else:
elif version == 3:
getPortsV3App(app_yml, app)
with open(app_cache_path, 'w') as f:
json.dump(createCleanConfigFromV3(app_yml, os.path.dirname(app_path)), f)
elif version == 4:
getPortsV4App(app_yml, app)
except Exception as e:
print(e)
print(traceback.format_exc())
print("App {} is invalid!".format(app))
appPortsToMap()
return {
"virtual_apps": virtual_apps,
"metadata": app_metadata,
"ports": appPortMap
"ports": appPortMap,
"portCache": appPorts,
}
citadelPorts = [
@ -91,31 +100,33 @@ citadelPorts = [
lastPort = 3000
def getNewPort(usedPorts):
def getNewPort(usedPorts, appId, containerName, allowExisting):
lastPort2 = lastPort
while lastPort2 in usedPorts or lastPort2 in citadelPorts:
while lastPort2 in usedPorts.keys() or lastPort2 in citadelPorts:
if allowExisting and lastPort2 in usedPorts.keys() and usedPorts[lastPort2]["app"] == appId and usedPorts[lastPort2]["container"] == containerName:
break
lastPort2 = lastPort2 + 1
return lastPort2
def validatePort(appContainer, port, appId, priority: int, isDynamic = False):
def validatePort(containerName, appContainer, port, appId, priority: int, isDynamic = False):
if port not in appPorts and port not in citadelPorts and port != 0:
appPorts[port] = {
"app": appId,
"port": port,
"container": appContainer["name"],
"container": containerName,
"priority": priority,
"dynamic": isDynamic,
}
else:
if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != appContainer["name"]:
newPort = getNewPort(appPorts.keys())
if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != containerName:
if port in appPorts and priority > appPorts[port]["priority"]:
#print("Prioritizing app {} over {}".format(appId, appPorts[port]["app"]))
newPort = getNewPort(appPorts, appPorts[port]["app"], appPorts[port]["container"], False)
appPorts[newPort] = appPorts[port].copy()
appPorts[port] = {
"app": appId,
"port": port,
"container": appContainer["name"],
"container": containerName,
"priority": priority,
"dynamic": isDynamic,
}
@ -124,11 +135,12 @@ def validatePort(appContainer, port, appId, priority: int, isDynamic = False):
disabledApps.append(appId)
print("App {} disabled because of port conflict".format(appId))
else:
#print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, appContainer["name"], newPort, port))
newPort = getNewPort(appPorts, appId, containerName, True)
#print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, containerName, newPort, port))
appPorts[newPort] = {
"app": appId,
"port": port,
"container": appContainer["name"],
"container": containerName,
"priority": priority,
"dynamic": isDynamic,
}
@ -136,28 +148,45 @@ def validatePort(appContainer, port, appId, priority: int, isDynamic = False):
def getPortsOldApp(app, appId):
for appContainer in app["containers"]:
if "port" in appContainer:
validatePort(appContainer, appContainer["port"], appId, 0)
validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0)
if "ports" in appContainer:
for port in appContainer["ports"]:
realPort = int(str(port).split(":")[0])
validatePort(appContainer, realPort, appId, 2)
validatePort(appContainer["name"], appContainer, realPort, appId, 2)
def getPortsV3App(app, appId):
for appContainer in app["containers"]:
if "port" in appContainer:
if "preferredOutsidePort" in appContainer and "requiresPort" in appContainer and appContainer["requiresPort"]:
validatePort(appContainer, appContainer["preferredOutsidePort"], appId, 2)
validatePort(appContainer["name"], appContainer, appContainer["preferredOutsidePort"], appId, 2)
elif "preferredOutsidePort" in appContainer:
validatePort(appContainer, appContainer["preferredOutsidePort"], appId, 1)
validatePort(appContainer["name"], appContainer, appContainer["preferredOutsidePort"], appId, 1)
else:
validatePort(appContainer, appContainer["port"], appId, 0)
validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0)
elif "requiredPorts" not in appContainer and "requiredUdpPorts" not in appContainer:
validatePort(appContainer, getNewPort(appPorts.keys()), appId, 0, True)
# if the container does not define a port, assume 3000, and pass it to the container as env var
validatePort(appContainer["name"], appContainer, 3000, appId, 0, True)
if "requiredPorts" in appContainer:
for port in appContainer["requiredPorts"]:
validatePort(appContainer, port, appId, 2)
validatePort(appContainer["name"], appContainer, port, appId, 2)
if "requiredUdpPorts" in appContainer:
for port in appContainer["requiredUdpPorts"]:
validatePort(appContainer, port, appId, 2)
validatePort(appContainer["name"], appContainer, port, appId, 2)
def getPortsV4App(app, appId):
for appContainerName in app["services"].keys():
appContainer = app["services"][appContainerName]
if "enable_networking" in appContainer and not appContainer["enable_networking"]:
return
assignIpV4(appId, appContainerName)
if "port" in appContainer:
validatePort(appContainerName, appContainer, appContainer["port"], appId, 0)
if "required_ports" in appContainer:
if "tcp" in appContainer["required_ports"]:
for port in appContainer["required_ports"]["tcp"].keys():
validatePort(appContainerName, appContainer, port, appId, 2)
if "udp" in appContainer["required_ports"]:
for port in appContainer["required_ports"]["udp"].keys():
validatePort(appContainerName, appContainer, port, appId, 2)

43
app/lib/validate.py
View File

@ -6,34 +6,29 @@ import os
import yaml
from jsonschema import validate
import yaml
import traceback
scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
nodeRoot = os.path.join(scriptDir, "..")
with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f:
schemaVersion1 = yaml.safe_load(f)
with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f:
schemaVersion2 = yaml.safe_load(f)
with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f:
schemaVersion3 = yaml.safe_load(f)
with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file:
dependencies = yaml.safe_load(file)
# Validates app data
# Returns true if valid, false otherwise
def validateApp(app: dict):
if 'version' in app and str(app['version']) == "1":
try:
validate(app, schemaVersion1)
return True
# Catch and log any errors, and return false
except Exception as e:
print(e)
return False
elif 'version' in app and str(app['version']) == "2":
if 'version' in app and str(app['version']) == "2":
try:
validate(app, schemaVersion2)
return True
# Catch and log any errors, and return false
except Exception as e:
print(e)
print(traceback.format_exc())
return False
elif 'version' in app and str(app['version']) == "3":
try:
@ -41,12 +36,13 @@ def validateApp(app: dict):
return True
# Catch and log any errors, and return false
except Exception as e:
print(e)
print(traceback.format_exc())
return False
else:
elif 'version' not in app and 'citadel_version' not in app:
print("Unsupported app version")
return False
else:
return True
# Read in an app.yml file and pass it to the validation function
# Returns true if valid, false otherwise
@ -72,14 +68,20 @@ def findApps(dir: str):
def findAndValidateApps(dir: str):
apps = []
app_data = {}
for root, dirs, files in os.walk(dir, topdown=False):
for name in dirs:
app_dir = os.path.join(root, name)
for subdir in os.scandir(dir):
if not subdir.is_dir():
continue
app_dir = subdir.path
if os.path.isfile(os.path.join(app_dir, "app.yml.jinja")):
os.chown(app_dir, 1000, 1000)
os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli preprocess --app-name '{}' /apps/{}/app.yml.jinja /apps/{}/app.yml --services 'lnd'".format(dir, dependencies['app-cli'], subdir.name, subdir.name, subdir.name))
if os.path.isfile(os.path.join(app_dir, "app.yml")):
apps.append(name)
apps.append(subdir.name)
# Read the app.yml and append it to app_data
with open(os.path.join(app_dir, "app.yml"), 'r') as f:
app_data[name] = yaml.safe_load(f)
app_data[subdir.name] = yaml.safe_load(f)
else:
print("App {} has no app.yml".format(subdir.name))
# Now validate all the apps using the validateAppFile function by passing the app.yml as an argument to it, if an app is invalid, remove it from the list
for app in apps:
appyml = app_data[app]
@ -113,6 +115,7 @@ def findAndValidateApps(dir: str):
should_continue=False
if not should_continue:
continue
if 'containers' in appyml:
for container in appyml['containers']:
if 'permissions' in container:
for permission in container['permissions']:

5
db/dependencies.yml Normal file
View File

@ -0,0 +1,5 @@
compose: v2.10.2
dashboard: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e
manager: ghcr.io/runcitadel/manager:v0.0.17@sha256:ba436a07d6f96282217851756d8c81aeaa03c42dfa2246a89a78fc3384eed3cb
middleware: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae
app-cli: ghcr.io/runcitadel/app-cli:main@sha256:6dad26faf652b930cb219a6261af8edfa84d5db4d679153700dbc1b136267bbf

13
docker-compose.yml
View File

@ -51,7 +51,6 @@ services:
image: nginx:1.21.6@sha256:2834dc507516af02784808c5f48b7cbe38b8ed5d0f4837f16e78d00deb7e7767
depends_on:
- dashboard
- manager
volumes:
- ${PWD}/nginx:/etc/nginx
restart: on-failure
@ -99,8 +98,7 @@ services:
default:
ipv4_address: $LND_IP
dashboard:
container_name: dashboard
image: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5
image: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e
restart: on-failure
stop_grace_period: 1m30s
networks:
@ -108,7 +106,7 @@ services:
ipv4_address: $DASHBOARD_IP
manager:
container_name: manager
image: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce
image: ghcr.io/runcitadel/manager:v0.0.17@sha256:ba436a07d6f96282217851756d8c81aeaa03c42dfa2246a89a78fc3384eed3cb
depends_on:
- tor
- redis
@ -162,13 +160,12 @@ services:
ipv4_address: $MANAGER_IP
middleware:
container_name: middleware
image: ghcr.io/runcitadel/middleware:v0.0.11@sha256:e472da8cbfa67d9a9dbf321334fe65cdf20a0f9b6d6bab33fdf07210f54e7002
image: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae
depends_on:
- manager
- bitcoin
- lightning
- redis
command: sh -c "./wait-for-manager.sh $MANAGER_IP && ./start.sh"
command: sh -c "./wait-for-manager.sh $MANAGER_IP && ././start.sh"
restart: on-failure
volumes:
- ${PWD}/lnd:/lnd
@ -223,6 +220,7 @@ services:
ipv4_address: $ELECTRUM_IP
redis:
container_name: redis
user: 1000:1000
image: redis:7.0.0-bullseye@sha256:ad0705f2e2344c4b642449e658ef4669753d6eb70228d46267685045bf932303
working_dir: /data
volumes:
@ -234,6 +232,7 @@ services:
networks:
default:
ipv4_address: $REDIS_IP
networks:
default:
name: citadel_main_network

25
events/triggers/quick-update Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
RELEASE=$(cat "$CITADEL_ROOT"/statuses/update-status.json | jq .updateTo -r)
cat <<EOF > "$CITADEL_ROOT"/statuses/update-status.json
{"state": "installing", "progress": 30, "description": "Starting update", "updateTo": "$RELEASE"}
EOF
curl "https://raw.githubusercontent.com/runcitadel/core/${RELEASE}/db/dependencies.yml" > "$CITADEL_ROOT"/db/dependencies
cat <<EOF > "$CITADEL_ROOT"/statuses/update-status.json
{"state": "installing", "progress": 70, "description": "Starting new containers", "updateTo": "$RELEASE"}
EOF
"${CITADEL_ROOT}/scripts/start"
cat <<EOF > "$CITADEL_ROOT"/statuses/update-status.json
{"state": "success", "progress": 100, "description": "Successfully installed Citadel $RELEASE", "updateTo": ""}
EOF

1
events/triggers/set-update-channel
View File

@ -7,3 +7,4 @@
CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
"${CITADEL_ROOT}/scripts/set-update-channel" "${1}"
"${CITADEL_ROOT}/scripts/start"

9
info.json
View File

@ -1,6 +1,7 @@
{
"version": "0.0.7",
"name": "Citadel 0.0.7",
"requires": ">=0.0.1",
"notes": "While we are busy with the next huge update, you may need to wait longer for updates. This update updates Bitcoin Knots and LND to their latest versions to ensure apps can utilize their latest features. In addition, this update includes the Citadel CLI. More information on that will be published soon."
"version": "0.0.8",
"name": "Citadel 0.0.8",
"requires": ">=0.0.5",
"isQuickUpdate": false,
"notes": "This update includes a new version of the Citadel app system and includes a few new apps. With the new app system, Citadel can handle more apps and makes app development eaier."
}

39
scripts/configure vendored
View File

@ -31,13 +31,14 @@ if not is_arm64 and not is_amd64:
print('Citadel only works on arm64 and amd64!')
exit(1)
dependencies = False
# Check the output of "docker compose version", if it matches "Docker Compose version v2.0.0-rc.3", return true
# Otherwise, return false
def is_compose_rc_or_outdated():
def is_compose_version_except(target_version):
try:
output = subprocess.check_output(['docker', 'compose', 'version'])
if output.decode('utf-8').strip() != 'Docker Compose version v2.3.3':
print("Using outdated Docker Compose, updating...")
if output.decode('utf-8').strip() != 'Docker Compose version {}'.format(target_version):
return True
else:
return False
@ -48,17 +49,19 @@ def is_compose_rc_or_outdated():
def download_docker_compose():
# Skip if os.path.expanduser('~/.docker/cli-plugins/docker-compose') exists
subprocess.check_call(["mkdir", "-p", os.path.expanduser('~/.docker/cli-plugins/')])
if (os.path.exists(os.path.expanduser('~/.docker/cli-plugins/docker-compose')) or os.path.exists('/usr/lib/docker/cli-plugins/docker-compose')) and not is_compose_rc_or_outdated():
print("Found {}\n".format(subprocess.check_output(['docker', 'compose', 'version']).decode('utf-8').strip()))
return
print("Installing Docker Compose...\n")
if is_arm64:
subprocess.check_call(['wget', 'https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-linux-aarch64', '-O', os.path.expanduser('~/.docker/cli-plugins/docker-compose')])
compose_arch = 'aarch64'
elif is_amd64:
subprocess.check_call(['wget', 'https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-linux-x86_64', '-O', os.path.expanduser('~/.docker/cli-plugins/docker-compose')])
os.chmod(os.path.expanduser('~/.docker/cli-plugins/docker-compose'), 0o755)
compose_arch = 'x86_64'
# We validate that no other case than the two above can happen before
if is_compose_version_except(dependencies['compose']):
print("Docker compose not found or not required version, updating.")
compose_url = 'https://github.com/docker/compose/releases/download/{}/docker-compose-linux-{}'.format(dependencies['compose'], compose_arch)
compose_file = os.path.expanduser('~/.docker/cli-plugins/docker-compose')
subprocess.check_call(['wget', compose_url, '-O', compose_file])
os.chmod(compose_file, 0o755)
if not shutil.which("wget"):
print('Wget is not installed!')
@ -72,6 +75,9 @@ if not shutil.which("docker"):
CITADEL_ROOT=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.chdir(CITADEL_ROOT)
with open("./db/dependencies.yml", "r") as file:
dependencies = yaml.safe_load(file)
updating = False
status_dir = os.path.join(CITADEL_ROOT, 'statuses')
# Make sure to use the main status dir for updates
@ -365,6 +371,15 @@ print("Generated configuration files\n")
print("Checking if Docker Compose is installed...")
download_docker_compose()
print("Updating core services...")
print()
with open("docker-compose.yml", 'r') as stream:
compose = yaml.safe_load(stream)
for service in ["manager", "middleware", "dashboard"]:
compose["services"][service]["image"] = dependencies[service]
with open("docker-compose.yml", "w") as stream:
yaml.dump(compose, stream, sort_keys=False)
if not reconfiguring:
print("Updating apps...\n")
os.system('./scripts/app --invoked-by-configure update')

2
scripts/set-update-channel
View File

@ -10,7 +10,7 @@ NODE_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)"
# If $1 is not given, fail
if [ -z "$1" ]; then
echo "Usage: $0 <channel>"
echo "Channel can currently either be 'stable' or 'beta'"
echo "Channel can currently either be 'stable', 'beta' or 'c-lightning'"
exit 1
fi
sed -i "s/UPDATE_CHANNEL=.*/UPDATE_CHANNEL=${1}/" "${NODE_ROOT}/.env"

1
scripts/update/.updateinclude
View File

@ -9,3 +9,4 @@ apps/docker-compose.common.yml
services/bitcoin/*
services/electrum/*
services/lightning/*
db/dependencies.yml