heierbtc/citadel-core
1
0
Fork 0
forked from michael.heier/citadel-core
Code Issues Pull Requests Projects Releases Wiki Activity

App system cleanups (#51)

Browse Source 
* Add app cli to docker-compose.yml

* Remove app.yml v1

* Add missing import

* More cleanups

* Another missing import

* Add mount for apps

* Remove more
This commit is contained in:
Aaron Dewes 2022-06-19 02:05:22 +02:00 committed by GitHub
parent ceb77e8bdd
commit faf6d62e42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 151 additions and 975 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

230
app-standard-v1.json
View File

@ -1,230 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Citadel app.yml v1",
"description": "The first draft of Citadel's app.yml format",
"type": "object",
"properties": {
"version": {
"type": [
"string",
"number"
],
"description": "The version of the app.yml format you're using."
},
"metadata": {
"type": "object",
"properties": {
"name": {
"description": "Displayed name of the app",
"type": "string"
},
"version": {
"description": "Displayed version for the app",
"type": "string"
},
"category": {
"description": "The category you'd put the app in",
"type": "string"
},
"tagline": {
"description": "A clever tagline",
"type": "string"
},
"description": {
"description": "A longer description of the app",
"type": "string"
},
"developer": {
"description": "The awesome people behind the app",
"type": "string"
},
"website": {
"description": "Displayed version for the app",
"type": "string"
},
"dependencies": {
"description": "The services the app depends on",
"type": "array",
"items": {
"type": "string"
}
},
"repo": {
"description": "The development repository for your app",
"type": "string"
},
"support": {
"description": "A link to the app support wiki/chat/...",
"type": "string"
},
"gallery": {
"type": "array",
"description": "URLs or paths in the runcitadel/app-images/[app-name] folder with app images",
"items": {
"type": "string"
}
},
"path": {
"description": "The path of the app's visible site the open button should open",
"type": "string"
},
"defaultPassword": {
"description": "The app's default password",
"type": "string"
},
"torOnly": {
"description": "Whether the app is only available over tor",
"type": "boolean"
},
"mainContainer": {
"type": "string",
"description": "The name of the main container for the app. If set, IP, port, and hidden service will be assigned to it automatically."
},
"updateContainer": {
"type": "string",
"description": "The container the developer system should automatically update."
}
},
"required": [
"name",
"version",
"category",
"tagline",
"description",
"developer",
"website",
"repo",
"support",
"gallery"
],
"additionalProperties": false
},
"containers": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"image": {
"type": "string"
},
"permissions": {
"type": "array",
"items": {
"type": "string",
"enum": [
"lnd",
"bitcoind",
"electrum",
"root",
"hw"
]
}
},
"ports": {
"type": "array",
"items": {
"type": [
"string",
"number"
]
}
},
"port": {
"type": "number",
"description": "If this is the main container, the port inside the container which will be exposed to the outside as the port specified in metadata."
},
"environment": {
"type": "object"
},
"data": {
"type": "array",
"description": "An array of at directories in the container the app stores its data in. Can be empty. Please only list top-level directories.",
"items": {
"type": "string"
}
},
"user": {
"type": "string",
"description": "The user the container should run as"
},
"stop_grace_period": {
"type": "string",
"description": "The grace period for stopping the container. Defaults to 1 minute."
},
"depends_on": {
"type": "array",
"description": "The services the container depends on"
},
"entrypoint": {
"type": [
"string",
"array"
],
"description": "The entrypoint for the container"
},
"bitcoin_mount_dir": {
"type": "string",
"description": "Where to mount the bitcoin dir"
},
"command": {
"type": [
"string",
"array"
],
"description": "The command for the container"
},
"init": {
"type": "boolean",
"description": "Whether the container should be run with init"
},
"stop_signal": {
"type": "string",
"description": "The signal to send to the container when stopping"
},
"noNetwork": {
"type": "boolean",
"description": "Set this to true if the container shouldn't get an IP & port exposed."
},
"needsHiddenService": {
"type": "boolean",
"description": "Set this to true if the container should be assigned a hidden service even if it's not the main container."
},
"hiddenServicePort": {
"type": "number",
"description": "Set this to a port if your container exposes multiple ports, but only one should be a hidden service."
},
"hiddenServicePorts": {
"type": "object",
"description": "Set this to a map of service names to hidden service ports if your container exposes multiple ports, and all of them should be hidden services.",
"patternProperties": {
"^[a-zA-Z0-9_]+$": {
"type": [
"number",
"array"
]
}
}
},
"restart": {
"type": "string",
"description": "When the container should restart. Can be 'always' or 'on-failure'."
}
},
"additionalProperties": false,
"required": [
"name",
"image"
]
},
"additionalProperties": false
}
},
"required": [
"metadata",
"containers"
],
"additionalProperties": false
}

3
app-standard-v1.json.license
View File

@ -1,3 +0,0 @@
SPDX-FileCopyrightText: 2021 Citadel and contributors
SPDX-License-Identifier: GPL-3.0-or-later

194
app/app-standard-v1.yml
View File

@ -1,194 +0,0 @@
# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema
$schema: https://json-schema.org/draft/2020-12/schema
title: Citadel app.yml v1
description: The first draft of Citadel's app.yml format
type: object
properties:
version:
type:
- string
- number
description: The version of the app.yml format you're using.
metadata:
type: object
properties:
name:
description: Displayed name of the app
type: string
version:
description: Displayed version for the app
type: string
category:
description: The category you'd put the app in
type: string
tagline:
description: A clever tagline
type: string
description:
description: A longer description of the app
type: string
developer:
description: The awesome people behind the app
type: string
website:
description: Displayed version for the app
type: string
dependencies:
description: The services the app depends on
type: array
items:
type: string
repo:
description: The development repository for your app
type: string
support:
description: A link to the app support wiki/chat/...
type: string
gallery:
type: array
description: >-
URLs or paths in the runcitadel/app-images/[app-name] folder with app
images
items:
type: string
path:
description: The path of the app's visible site the open button should open
type: string
defaultPassword:
description: The app's default password
type: string
torOnly:
description: Whether the app is only available over tor
type: boolean
mainContainer:
type: string
description: >-
The name of the main container for the app. If set, IP, port, and
hidden service will be assigned to it automatically.
updateContainer:
type: string
description: The container the developer system should automatically update.
required:
- name
- version
- category
- tagline
- description
- developer
- website
- repo
- support
- gallery
additionalProperties: false
containers:
type: array
items:
type: object
properties:
name:
type: string
image:
type: string
permissions:
type: array
items:
type: string
enum:
- lnd
- bitcoind
- electrum
- root
- hw
ports:
type: array
items:
type:
- string
- number
port:
type: number
description: >-
If this is the main container, the port inside the container which
will be exposed to the outside as the port specified in metadata.
environment:
type: object
data:
type: array
description: >-
An array of at directories in the container the app stores its data
in. Can be empty. Please only list top-level directories.
items:
type: string
user:
type: string
description: The user the container should run as
stop_grace_period:
type: string
description: The grace period for stopping the container. Defaults to 1 minute.
depends_on:
type: array
description: The services the container depends on
entrypoint:
type:
- string
- array
description: The entrypoint for the container
bitcoin_mount_dir:
type: string
description: Where to mount the bitcoin dir
command:
type:
- string
- array
description: The command for the container
init:
type: boolean
description: Whether the container should be run with init
stop_signal:
type: string
description: The signal to send to the container when stopping
noNetwork:
type: boolean
description: >-
Set this to true if the container shouldn't get an IP & port
exposed.
needsHiddenService:
type: boolean
description: >-
Set this to true if the container should be assigned a hidden
service even if it's not the main container.
hiddenServicePort:
type: number
description: >-
Set this to a port if your container exposes multiple ports, but
only one should be a hidden service.
hiddenServicePorts:
type: object
description: >-
Set this to a map of service names to hidden service ports if your
container exposes multiple ports, and all of them should be hidden
services.
patternProperties:
^[a-zA-Z0-9_]+$:
type:
- number
- array
restart:
type: string
description: When the container should restart. Can be 'always' or 'on-failure'.
additionalProperties: false
required:
- name
- image
additionalProperties: false
required:
- metadata
- containers
additionalProperties: false

2
app/lib/composegenerator/shared/env.py
View File

@ -4,7 +4,7 @@
import re
from typing import Union
from lib.composegenerator.v1.types import App
from lib.composegenerator.v2.types import App
from lib.composegenerator.shared.const import always_allowed_env
from lib.citadelutils import checkArrayContainsAllElements, getEnvVars

2
app/lib/composegenerator/shared/main.py
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
# Main functions
from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container
from lib.composegenerator.v2.types import App, AppStage3, AppStage2, Container
from lib.composegenerator.shared.const import permissions

137
app/lib/composegenerator/shared/networking.py Normal file
View File

@ -0,0 +1,137 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
import json
from os import path
import random
from lib.composegenerator.v2.types import ContainerStage2, NetworkConfig
from lib.citadelutils import parse_dotenv
from dacite import from_dict
def getFreePort(networkingFile: str, appId: str):
# Ports used currently in Citadel
usedPorts = [
# Dashboard
80,
# Sometimes used by nginx with some setups
433,
# Dashboard SSL
443,
# Bitcoin Core P2P
8333,
# LND gRPC
10009,
# LND REST
8080,
# Electrum Server
50001,
# Tor Proxy
9050,
]
networkingData = {}
if path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ports' in networkingData:
usedPorts += list(networkingData['ports'].values())
else:
networkingData['ports'] = {}
if appId in networkingData['ports']:
return networkingData['ports'][appId]
while True:
port = str(random.randint(1024, 49151))
if port not in usedPorts:
# Check if anyhing is listening on the specific port
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
networkingData['ports'][appId] = port
break
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
return port
def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
# Strip leading/trailing whitespace from container.name
container.name = container.name.strip()
# If the name still contains a newline, throw an error
if container.name.find("\n") != -1:
raise Exception("Newline in container name")
env_var = "APP_{}_{}_IP".format(
appId.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
# can be used
usedIps = []
networkingData = {}
if path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ip_addresses' in networkingData:
usedIps = list(networkingData['ip_addresses'].values())
else:
networkingData['ip_addresses'] = {}
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
# If the IP is already in use, it will be tried again until it's not in use
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
# Throw an error, because no more IPs can be used
if len(usedIps) == 235:
raise Exception("No more IPs can be used")
if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
ip = networkingData['ip_addresses']["{}-{}".format(
appId, container.name)]
else:
while True:
ip = "10.21.21." + str(random.randint(20, 255))
if ip not in usedIps:
networkingData['ip_addresses']["{}-{}".format(
appId, container.name)] = ip
break
container.networks = from_dict(data_class=NetworkConfig, data={'default': {
'ipv4_address': "$" + env_var}})
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
return container
# Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, ip))
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
return container
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
# Strip leading/trailing whitespace from container.name
container.name = container.name.strip()
# If the name still contains a newline, throw an error
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
raise Exception("Newline or space in container name")
env_var = "APP_{}_{}_PORT".format(
appId.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
port = getFreePort(networkingFile, appId)
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
return {"port": port, "env_var": "${{{}}}".format(env_var)}
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, port))
# This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
# where the outer {{ }} will be replaced by {} in the returned string
return {"port": port, "env_var": "${{{}}}".format(env_var)}

30
app/lib/composegenerator/v1/generate.py
View File

@ -1,30 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
from lib.composegenerator.v1.types import App, AppStage4, generateApp
from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
from lib.composegenerator.shared.env import validateEnv
from lib.citadelutils import classToDict
import os
def createComposeConfigFromV1(app: dict, nodeRoot: str):
envFile = os.path.join(nodeRoot, ".env")
networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
newApp: App = generateApp(app)
newApp = convertContainerPermissions(newApp)
validateEnv(newApp)
newApp = convertDataDirToVolume(newApp)
newApp = configureIps(newApp, networkingFile, envFile)
newApp = configureMainPort(newApp, nodeRoot)
configureHiddenServices(newApp, nodeRoot)
finalConfig: AppStage4 = convertContainersToServices(newApp)
newApp = classToDict(finalConfig)
del newApp['metadata']
if "version" in newApp:
del newApp["version"]
# Set version to 3.8 (current compose file version)
newApp = {'version': '3.8', **newApp}
return newApp

226
app/lib/composegenerator/v1/networking.py
View File

@ -1,226 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
from dacite import from_dict
from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig
from lib.citadelutils import parse_dotenv
import json
from os import path
import random
from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService
def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
# Strip leading/trailing whitespace from container.name
container.name = container.name.strip()
# If the name still contains a newline, throw an error
if container.name.find("\n") != -1:
raise Exception("Newline in container name")
env_var = "APP_{}_{}_IP".format(
appId.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
# can be used
usedIps = []
networkingData = {}
if path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ip_addresses' in networkingData:
usedIps = list(networkingData['ip_addresses'].values())
else:
networkingData['ip_addresses'] = {}
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
# If the IP is already in use, it will be tried again until it's not in use
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
# Throw an error, because no more IPs can be used
if len(usedIps) == 235:
raise Exception("No more IPs can be used")
if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
ip = networkingData['ip_addresses']["{}-{}".format(
appId, container.name)]
else:
while True:
ip = "10.21.21." + str(random.randint(20, 255))
if ip not in usedIps:
networkingData['ip_addresses']["{}-{}".format(
appId, container.name)] = ip
break
container.networks = from_dict(data_class=NetworkConfig, data={'default': {
'ipv4_address': "$" + env_var}})
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
return container
# Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, ip))
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
return container
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
# Strip leading/trailing whitespace from container.name
container.name = container.name.strip()
# If the name still contains a newline, throw an error
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
raise Exception("Newline or space in container name")
env_var = "APP_{}_{}_PORT".format(
appId.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
port = getFreePort(networkingFile, appId)
dotEnv = parse_dotenv(envFile)
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
return {"port": port, "env_var": "${{{}}}".format(env_var)}
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
with open(envFile, 'a') as f:
f.write("{}={}\n".format(env_var, port))
# This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
# where the outer {{ }} will be replaced by {} in the returned string
return {"port": port, "env_var": "${{{}}}".format(env_var)}
def getMainContainer(app: dict):
if len(app.containers) == 1:
return app.containers[0]
else:
if not app.metadata.mainContainer:
app.metadata.mainContainer = 'main'
for container in app.containers:
if container.name == app.metadata.mainContainer:
return container
raise Exception(
"No main container found for app {}".format(app.metadata.name))
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
registryFile = path.join(nodeRoot, "apps", "registry.json")
registry: list = []
if path.isfile(registryFile):
with open(registryFile, 'r') as f:
registry = json.load(f)
else:
raise Exception("Registry file not found")
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
mainContainer = getMainContainer(app)
portDetails = assignPort(mainContainer, app.metadata.id, path.join(
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
containerPort = portDetails['port']
portAsEnvVar = portDetails['env_var']
portToAppend = portAsEnvVar
mainPort = False
if mainContainer.port:
portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port)
mainPort = mainContainer.port
del mainContainer.port
else:
portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar)
if mainContainer.ports:
mainContainer.ports.append(portToAppend)
# Set the main port to the first port in the list, if it contains a :, it's the port after the :
# If it doesn't contain a :, it's the port itself
if mainPort == False:
mainPort = mainContainer.ports[0]
if mainPort.find(":") != -1:
mainPort = mainPort.split(":")[1]
else:
mainContainer.ports = [portToAppend]
if mainPort == False:
mainPort = portDetails['port']
mainContainer = assignIp(mainContainer, app.metadata.id, path.join(
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
# If the IP wasn't in dotenv before, now it should be
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace(
"-", "_"), mainContainer.name.upper().replace("-", "_"))]
hiddenservice = getHiddenService(
app.metadata.name, app.metadata.id, containerIP, mainPort)
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(hiddenservice)
# Also set the port in metadata
app.metadata.port = int(containerPort)
for registryApp in registry:
if registryApp['id'] == app.metadata.id:
registry[registry.index(registryApp)]['port'] = int(containerPort)
break
with open(registryFile, 'w') as f:
json.dump(registry, f, indent=4, sort_keys=True)
return app
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
for container in app.containers:
if container.noNetwork:
# Check if port is defined for the container
if container.port:
raise Exception("Port defined for container without network")
if app.metadata.mainContainer == container.name:
raise Exception("Main container without network")
# Skip this iteration of the loop
continue
container = assignIp(container, app.metadata.id,
networkingFile, envFile)
return app
def configureHiddenServices(app: dict, nodeRoot: str) -> None:
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
hiddenServices = ""
if len(app.containers) == 1:
mainContainer = app.containers[0]
else:
mainContainer = None
if app.metadata.mainContainer == None:
app.metadata.mainContainer = 'main'
for container in app.containers:
if container.name == app.metadata.mainContainer:
mainContainer = container
break
if mainContainer is None:
raise Exception("No main container found")
for container in app.containers:
env_var = "APP_{}_{}_IP".format(
app.metadata.id.upper().replace("-", "_"),
container.name.upper().replace("-", "_")
)
hiddenServices += getContainerHiddenService(
app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name)
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
f.write(hiddenServices)

151
app/lib/composegenerator/v1/types.py
View File

@ -1,151 +0,0 @@
from typing import Union, List
from dataclasses import dataclass, field
from dacite import from_dict
@dataclass
class Metadata:
id: str
name: str
version: str
category: str
tagline: str
description: str
developer: str
website: str
repo: str
support: str
gallery: List[str] = field(default_factory=list)
dependencies: List[str] = field(default_factory=list)
mainContainer: Union[str, None] = None
updateContainer: Union[str, None] = None
path: str = ""
defaultPassword: str = ""
torOnly: bool = False
@dataclass
class Container:
name: str
image: str
permissions: list = field(default_factory=list)
ports: list = field(default_factory=list)
port: Union[int, None] = None
environment: Union[dict, None] = None
data: list = field(default_factory=list)
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: list = field(default_factory=list)
entrypoint: Union[List[str], str] = field(default_factory=list)
bitcoin_mount_dir: Union[str, None] = None
command: Union[List[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
environment_allow: list = field(default_factory=list)
# Only added later
volumes: list = field(default_factory=list)
restart: Union[str, None] = None
@dataclass
class App:
version: Union[str, int]
metadata: Metadata
containers: List[Container]
# Generate an app instance from an app dict
def generateApp(appDict):
return from_dict(data_class=App, data=appDict)
@dataclass
class Network:
ipv4_address: Union[str, None] = None
@dataclass
class NetworkConfig:
default: Network
# After converting data dir and defining volumes, stage 2
@dataclass
class ContainerStage2:
id: str
name: str
image: str
permissions: List[str] = field(default_factory=list)
ports: list = field(default_factory=list)
environment: Union[dict, None] = None
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: List[str] = field(default_factory=list)
entrypoint: Union[List[str], str] = field(default_factory=list)
command: Union[List[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
volumes: List[str] = field(default_factory=list)
networks: NetworkConfig = field(default_factory=NetworkConfig)
restart: Union[str, None] = None
@dataclass
class AppStage2:
version: Union[str, int]
metadata: Metadata
containers: List[ContainerStage2]
@dataclass
class MetadataStage3:
id: str
name: str
version: str
category: str
tagline: str
description: str
developer: str
website: str
dependencies: List[str]
repo: str
support: str
gallery: List[str]
mainContainer: Union[str, None] = None
updateContainer: Union[str, None] = None
path: str = ""
defaultPassword: str = ""
torOnly: bool = False
@dataclass
class AppStage3:
version: Union[str, int]
metadata: MetadataStage3
containers: List[ContainerStage2]
@dataclass
class ContainerStage4:
id: str
name: str
image: str
ports: list = field(default_factory=list)
environment: Union[dict, None] = None
user: Union[str, None] = None
stop_grace_period: str = '1m'
depends_on: List[str] = field(default_factory=list)
entrypoint: Union[List[str], str] = field(default_factory=list)
command: Union[List[str], str] = field(default_factory=list)
init: Union[bool, None] = None
stop_signal: Union[str, None] = None
noNetwork: Union[bool, None] = None
needsHiddenService: Union[bool, None] = None
hiddenServicePort: Union[int, None] = None
hiddenServicePorts: Union[dict, None] = None
volumes: List[str] = field(default_factory=list)
networks: NetworkConfig = field(default_factory=NetworkConfig)
restart: Union[str, None] = None
@dataclass
class AppStage4:
version: Union[str, int]
metadata: MetadataStage3
services: List[ContainerStage4]

118
app/lib/composegenerator/v1/utils/networking.py
View File

@ -1,118 +0,0 @@
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later
import json
import os
import random
from lib.composegenerator.v1.types import Container
def getFreePort(networkingFile: str, appId: str):
# Ports used currently in Citadel
usedPorts = [
# Dashboard
80,
# Sometimes used by nginx with some setups
433,
# Dashboard SSL
443,
# Bitcoin Core P2P
8333,
# LND gRPC
10009,
# LND REST
8080,
# Electrum Server
50001,
# Tor Proxy
9050,
]
networkingData = {}
if os.path.isfile(networkingFile):
with open(networkingFile, 'r') as f:
networkingData = json.load(f)
if 'ports' in networkingData:
usedPorts += list(networkingData['ports'].values())
else:
networkingData['ports'] = {}
if appId in networkingData['ports']:
return networkingData['ports'][appId]
while True:
port = str(random.randint(1024, 49151))
if port not in usedPorts:
# Check if anyhing is listening on the specific port
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
networkingData['ports'][appId] = port
break
with open(networkingFile, 'w') as f:
json.dump(networkingData, f)
return port
def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str:
hiddenServices = '''
# {} Hidden Service
HiddenServiceDir /var/lib/tor/app-{}
'''.format(name, id)
for port in ports:
hiddenServices += 'HiddenServicePort {} {}:{}'.format(
port, internalIp, port)
hiddenServices += "\n"
return hiddenServices
def getHiddenServiceString(name: str, id: str, internalPort, internalIp: str, publicPort) -> str:
return '''
# {} Hidden Service
HiddenServiceDir /var/lib/tor/app-{}
HiddenServicePort {} {}:{}
'''.format(name, id, publicPort, internalIp, internalPort)
def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str:
return getHiddenServiceString(appName, appId, appPort, appIp, "80")
def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str:
if not container.needsHiddenService and not isMainContainer:
return ""
if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer:
print("Container {} for app {} isn't compatible with hidden service assignment".format(
container.name, appName))
return ""
if isMainContainer:
if not container.hiddenServicePorts:
return ""
# hiddenServicePorts is a map of hidden service name to port
# We need to generate a hidden service for each one
hiddenServices = ""
for name, port in container.hiddenServicePorts.items():
if ".." in name:
print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName))
# If port is a list, use getHiddenServiceMultiPort
if isinstance(port, list):
hiddenServices += getHiddenServiceMultiPort("{} {}".format(appName, name), "{}-{}".format(
appId, name), containerIp, port)
else:
hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format(
appId, name), port, containerIp, port)
del container.hiddenServicePorts
return hiddenServices
del container.needsHiddenService
if not container.port:
data = getHiddenServiceString(appName + container.name, "{}-{}".format(
appId, container.name), container.hiddenServicePort, containerIp, "80")
del container.hiddenServicePort
return data
else:
return getHiddenServiceString(appName + container.name, "{}-{}".format(
appId, container.name), container.port, containerIp, container.port)

2
app/lib/composegenerator/v2/networking.py
View File

@ -8,7 +8,7 @@ import json
from os import path
import random
from lib.composegenerator.v2.utils.networking import getContainerHiddenService
from lib.composegenerator.v1.networking import assignIp, assignPort
from lib.composegenerator.shared.networking import assignIp, assignPort
def getMainContainer(app: App) -> Container:

4
app/lib/composegenerator/v3/networking.py
View File

@ -7,8 +7,7 @@ from lib.citadelutils import parse_dotenv
import json
from os import path
import random
from lib.composegenerator.v1.networking import assignIp, assignPort
from lib.composegenerator.shared.networking import assignIp, assignPort
def getMainContainerIndex(app: App):
if len(app.containers) == 1:
@ -105,4 +104,3 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
with open(envFile, 'a') as f:
f.write("{}={}\n".format(portAsEnvVar, app.metadata.port))
return app

6
app/lib/manage.py
View File

@ -25,7 +25,6 @@ except Exception:
print("Continuing anyway, but some features won't be available,")
print("for example checking for app updates")
from lib.composegenerator.v1.generate import createComposeConfigFromV1
from lib.composegenerator.v2.generate import createComposeConfigFromV2
from lib.composegenerator.v3.generate import createComposeConfigFromV3
from lib.validate import findAndValidateApps
@ -188,10 +187,7 @@ def getApp(appFile: str, appId: str):
raise Exception("Error: Could not find metadata in " + appFile)
app["metadata"]["id"] = appId
if 'version' in app and str(app['version']) == "1":
print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId))
return createComposeConfigFromV1(app, nodeRoot)
elif 'version' in app and str(app['version']) == "2":
if 'version' in app and str(app['version']) == "2":
print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId))
return createComposeConfigFromV2(app, nodeRoot)
elif 'version' in app and str(app['version']) == "3":

2
app/lib/metadata.py
View File

@ -7,7 +7,7 @@ import yaml
from lib.composegenerator.next.stage1 import createCleanConfigFromV3
from lib.composegenerator.v2.networking import getMainContainer
from lib.composegenerator.v1.networking import getFreePort
from lib.composegenerator.shared.networking import getFreePort
from lib.entropy import deriveEntropy
from typing import List
import json

12
app/lib/validate.py
View File

@ -9,8 +9,6 @@ import yaml
scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f:
schemaVersion1 = yaml.safe_load(f)
with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f:
schemaVersion2 = yaml.safe_load(f)
with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f:
@ -19,15 +17,7 @@ with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f:
# Validates app data
# Returns true if valid, false otherwise
def validateApp(app: dict):
if 'version' in app and str(app['version']) == "1":
try:
validate(app, schemaVersion1)
return True
# Catch and log any errors, and return false
except Exception as e:
print(e)
return False
elif 'version' in app and str(app['version']) == "2":
if 'version' in app and str(app['version']) == "2":
try:
validate(app, schemaVersion2)
return True

7
docker-compose.yml
View File

@ -234,6 +234,13 @@ services:
networks:
default:
ipv4_address: $REDIS_IP
app-cli:
container_name: app-cli
image: ghcr.io/runcitadel/app-cli:main@sha256:694e52fa9da1ac976165f269c17e27803032a05a76293dfe3589a50813306ded
volumes:
- ${PWD}/apps:/apps
networks:
default:
name: citadel_main_network