mirror of
https://github.com/runcitadel/core.git
synced 2024-11-11 16:30:38 +00:00
Merge remote-tracking branch 'origin/release/0.0.8' into deno
This commit is contained in:
Aaron Dewes
commit
847bd3850e
|
|
@ -1,230 +0,0 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "Citadel app.yml v1",
|
||||
"description": "The first draft of Citadel's app.yml format",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"version": {
|
||||
"type": [
|
||||
"string",
|
||||
"number"
|
||||
],
|
||||
"description": "The version of the app.yml format you're using."
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "Displayed name of the app",
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"description": "Displayed version for the app",
|
||||
"type": "string"
|
||||
},
|
||||
"category": {
|
||||
"description": "The category you'd put the app in",
|
||||
"type": "string"
|
||||
},
|
||||
"tagline": {
|
||||
"description": "A clever tagline",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"description": "A longer description of the app",
|
||||
"type": "string"
|
||||
},
|
||||
"developer": {
|
||||
"description": "The awesome people behind the app",
|
||||
"type": "string"
|
||||
},
|
||||
"website": {
|
||||
"description": "Displayed version for the app",
|
||||
"type": "string"
|
||||
},
|
||||
"dependencies": {
|
||||
"description": "The services the app depends on",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"repo": {
|
||||
"description": "The development repository for your app",
|
||||
"type": "string"
|
||||
},
|
||||
"support": {
|
||||
"description": "A link to the app support wiki/chat/...",
|
||||
"type": "string"
|
||||
},
|
||||
"gallery": {
|
||||
"type": "array",
|
||||
"description": "URLs or paths in the runcitadel/app-images/[app-name] folder with app images",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"path": {
|
||||
"description": "The path of the app's visible site the open button should open",
|
||||
"type": "string"
|
||||
},
|
||||
"defaultPassword": {
|
||||
"description": "The app's default password",
|
||||
"type": "string"
|
||||
},
|
||||
"torOnly": {
|
||||
"description": "Whether the app is only available over tor",
|
||||
"type": "boolean"
|
||||
},
|
||||
"mainContainer": {
|
||||
"type": "string",
|
||||
"description": "The name of the main container for the app. If set, IP, port, and hidden service will be assigned to it automatically."
|
||||
},
|
||||
"updateContainer": {
|
||||
"type": "string",
|
||||
"description": "The container the developer system should automatically update."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"version",
|
||||
"category",
|
||||
"tagline",
|
||||
"description",
|
||||
"developer",
|
||||
"website",
|
||||
"repo",
|
||||
"support",
|
||||
"gallery"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"containers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"image": {
|
||||
"type": "string"
|
||||
},
|
||||
"permissions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"lnd",
|
||||
"bitcoind",
|
||||
"electrum",
|
||||
"root",
|
||||
"hw"
|
||||
]
|
||||
}
|
||||
},
|
||||
"ports": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": [
|
||||
"string",
|
||||
"number"
|
||||
]
|
||||
}
|
||||
},
|
||||
"port": {
|
||||
"type": "number",
|
||||
"description": "If this is the main container, the port inside the container which will be exposed to the outside as the port specified in metadata."
|
||||
},
|
||||
"environment": {
|
||||
"type": "object"
|
||||
},
|
||||
"data": {
|
||||
"type": "array",
|
||||
"description": "An array of at directories in the container the app stores its data in. Can be empty. Please only list top-level directories.",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"user": {
|
||||
"type": "string",
|
||||
"description": "The user the container should run as"
|
||||
},
|
||||
"stop_grace_period": {
|
||||
"type": "string",
|
||||
"description": "The grace period for stopping the container. Defaults to 1 minute."
|
||||
},
|
||||
"depends_on": {
|
||||
"type": "array",
|
||||
"description": "The services the container depends on"
|
||||
},
|
||||
"entrypoint": {
|
||||
"type": [
|
||||
"string",
|
||||
"array"
|
||||
],
|
||||
"description": "The entrypoint for the container"
|
||||
},
|
||||
"bitcoin_mount_dir": {
|
||||
"type": "string",
|
||||
"description": "Where to mount the bitcoin dir"
|
||||
},
|
||||
"command": {
|
||||
"type": [
|
||||
"string",
|
||||
"array"
|
||||
],
|
||||
"description": "The command for the container"
|
||||
},
|
||||
"init": {
|
||||
"type": "boolean",
|
||||
"description": "Whether the container should be run with init"
|
||||
},
|
||||
"stop_signal": {
|
||||
"type": "string",
|
||||
"description": "The signal to send to the container when stopping"
|
||||
},
|
||||
"noNetwork": {
|
||||
"type": "boolean",
|
||||
"description": "Set this to true if the container shouldn't get an IP & port exposed."
|
||||
},
|
||||
"needsHiddenService": {
|
||||
"type": "boolean",
|
||||
"description": "Set this to true if the container should be assigned a hidden service even if it's not the main container."
|
||||
},
|
||||
"hiddenServicePort": {
|
||||
"type": "number",
|
||||
"description": "Set this to a port if your container exposes multiple ports, but only one should be a hidden service."
|
||||
},
|
||||
"hiddenServicePorts": {
|
||||
"type": "object",
|
||||
"description": "Set this to a map of service names to hidden service ports if your container exposes multiple ports, and all of them should be hidden services.",
|
||||
"patternProperties": {
|
||||
"^[a-zA-Z0-9_]+$": {
|
||||
"type": [
|
||||
"number",
|
||||
"array"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"restart": {
|
||||
"type": "string",
|
||||
"description": "When the container should restart. Can be 'always' or 'on-failure'."
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"name",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"metadata",
|
||||
"containers"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
SPDX-FileCopyrightText: 2021 Citadel and contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema
|
||||
$schema: https://json-schema.org/draft/2020-12/schema
|
||||
|
||||
|
||||
title: Citadel app.yml v1
|
||||
description: The first draft of Citadel's app.yml format
|
||||
type: object
|
||||
|
||||
properties:
|
||||
version:
|
||||
type:
|
||||
- string
|
||||
- number
|
||||
description: The version of the app.yml format you're using.
|
||||
|
||||
metadata:
|
||||
type: object
|
||||
properties:
|
||||
name:
|
||||
description: Displayed name of the app
|
||||
type: string
|
||||
version:
|
||||
description: Displayed version for the app
|
||||
type: string
|
||||
category:
|
||||
description: The category you'd put the app in
|
||||
type: string
|
||||
tagline:
|
||||
description: A clever tagline
|
||||
type: string
|
||||
description:
|
||||
description: A longer description of the app
|
||||
type: string
|
||||
developer:
|
||||
description: The awesome people behind the app
|
||||
type: string
|
||||
website:
|
||||
description: Displayed version for the app
|
||||
type: string
|
||||
dependencies:
|
||||
description: The services the app depends on
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
repo:
|
||||
description: The development repository for your app
|
||||
type: string
|
||||
support:
|
||||
description: A link to the app support wiki/chat/...
|
||||
type: string
|
||||
gallery:
|
||||
type: array
|
||||
description: >-
|
||||
URLs or paths in the runcitadel/app-images/[app-name] folder with app
|
||||
images
|
||||
items:
|
||||
type: string
|
||||
path:
|
||||
description: The path of the app's visible site the open button should open
|
||||
type: string
|
||||
defaultPassword:
|
||||
description: The app's default password
|
||||
type: string
|
||||
torOnly:
|
||||
description: Whether the app is only available over tor
|
||||
type: boolean
|
||||
mainContainer:
|
||||
type: string
|
||||
description: >-
|
||||
The name of the main container for the app. If set, IP, port, and
|
||||
hidden service will be assigned to it automatically.
|
||||
updateContainer:
|
||||
type: string
|
||||
description: The container the developer system should automatically update.
|
||||
required:
|
||||
- name
|
||||
- version
|
||||
- category
|
||||
- tagline
|
||||
- description
|
||||
- developer
|
||||
- website
|
||||
- repo
|
||||
- support
|
||||
- gallery
|
||||
additionalProperties: false
|
||||
|
||||
containers:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
image:
|
||||
type: string
|
||||
permissions:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- lnd
|
||||
- bitcoind
|
||||
- electrum
|
||||
- root
|
||||
- hw
|
||||
ports:
|
||||
type: array
|
||||
items:
|
||||
type:
|
||||
- string
|
||||
- number
|
||||
port:
|
||||
type: number
|
||||
description: >-
|
||||
If this is the main container, the port inside the container which
|
||||
will be exposed to the outside as the port specified in metadata.
|
||||
environment:
|
||||
type: object
|
||||
data:
|
||||
type: array
|
||||
description: >-
|
||||
An array of at directories in the container the app stores its data
|
||||
in. Can be empty. Please only list top-level directories.
|
||||
items:
|
||||
type: string
|
||||
user:
|
||||
type: string
|
||||
description: The user the container should run as
|
||||
stop_grace_period:
|
||||
type: string
|
||||
description: The grace period for stopping the container. Defaults to 1 minute.
|
||||
depends_on:
|
||||
type: array
|
||||
description: The services the container depends on
|
||||
entrypoint:
|
||||
type:
|
||||
- string
|
||||
- array
|
||||
description: The entrypoint for the container
|
||||
bitcoin_mount_dir:
|
||||
type: string
|
||||
description: Where to mount the bitcoin dir
|
||||
command:
|
||||
type:
|
||||
- string
|
||||
- array
|
||||
description: The command for the container
|
||||
init:
|
||||
type: boolean
|
||||
description: Whether the container should be run with init
|
||||
stop_signal:
|
||||
type: string
|
||||
description: The signal to send to the container when stopping
|
||||
noNetwork:
|
||||
type: boolean
|
||||
description: >-
|
||||
Set this to true if the container shouldn't get an IP & port
|
||||
exposed.
|
||||
needsHiddenService:
|
||||
type: boolean
|
||||
description: >-
|
||||
Set this to true if the container should be assigned a hidden
|
||||
service even if it's not the main container.
|
||||
hiddenServicePort:
|
||||
type: number
|
||||
description: >-
|
||||
Set this to a port if your container exposes multiple ports, but
|
||||
only one should be a hidden service.
|
||||
hiddenServicePorts:
|
||||
type: object
|
||||
description: >-
|
||||
Set this to a map of service names to hidden service ports if your
|
||||
container exposes multiple ports, and all of them should be hidden
|
||||
services.
|
||||
patternProperties:
|
||||
^[a-zA-Z0-9_]+$:
|
||||
type:
|
||||
- number
|
||||
- array
|
||||
restart:
|
||||
type: string
|
||||
description: When the container should restart. Can be 'always' or 'on-failure'.
|
||||
additionalProperties: false
|
||||
required:
|
||||
- name
|
||||
- image
|
||||
additionalProperties: false
|
||||
|
||||
required:
|
||||
- metadata
|
||||
- containers
|
||||
|
||||
additionalProperties: false
|
||||
|
|
@ -3,6 +3,8 @@
|
|||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import re
|
||||
import fcntl
|
||||
import os
|
||||
|
||||
# Helper functions
|
||||
|
||||
|
|
@ -77,3 +79,27 @@ def classToDict(theClass):
|
|||
obj[key] = classToDict(value)
|
||||
return obj
|
||||
|
||||
class FileLock:
|
||||
"""Implements a file-based lock using flock(2).
|
||||
The lock file is saved in directory dir with name lock_name.
|
||||
dir is the current directory by default.
|
||||
"""
|
||||
|
||||
def __init__(self, lock_name, dir="."):
|
||||
self.lock_file = open(os.path.join(dir, lock_name), "w")
|
||||
|
||||
def acquire(self, blocking=True):
|
||||
"""Acquire the lock.
|
||||
If the lock is not already acquired, return None. If the lock is
|
||||
acquired and blocking is True, block until the lock is released. If
|
||||
the lock is acquired and blocking is False, raise an IOError.
|
||||
"""
|
||||
ops = fcntl.LOCK_EX
|
||||
if not blocking:
|
||||
ops |= fcntl.LOCK_NB
|
||||
fcntl.flock(self.lock_file, ops)
|
||||
|
||||
def release(self):
|
||||
"""Release the lock. Return None even if lock not currently acquired"""
|
||||
fcntl.flock(self.lock_file, fcntl.LOCK_UN)
|
||||
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
import re
|
||||
from typing import Union
|
||||
from lib.composegenerator.v1.types import App
|
||||
from lib.composegenerator.v2.types import App
|
||||
from lib.composegenerator.shared.const import always_allowed_env
|
||||
from lib.citadelutils import checkArrayContainsAllElements, getEnvVars
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# Main functions
|
||||
from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container
|
||||
from lib.composegenerator.v2.types import App, AppStage3
|
||||
from lib.composegenerator.shared.const import permissions
|
||||
|
||||
|
||||
|
|
@ -27,28 +27,3 @@ def convertContainersToServices(app: AppStage3) -> AppStage3:
|
|||
del app.containers
|
||||
app.services = services
|
||||
return app
|
||||
|
||||
# Converts the data of every container in app.containers to a volume, which is then added to the app
|
||||
def convertDataDirToVolume(app: App) -> AppStage2:
|
||||
for container in app.containers:
|
||||
# Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
|
||||
# Also, a datadir shouldn't start with a /
|
||||
for dataDir in container.data:
|
||||
if dataDir.find("..") == -1 and dataDir[0] != "/":
|
||||
container.volumes.append(
|
||||
'${APP_DATA_DIR}/' + dataDir)
|
||||
else:
|
||||
print("Data dir " + dataDir +
|
||||
" contains invalid characters")
|
||||
del container.data
|
||||
if container.bitcoin_mount_dir != None:
|
||||
if not 'bitcoind' in container.permissions:
|
||||
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
|
||||
# Skip this container
|
||||
continue
|
||||
# Also skip the container if container.bitcoin_mount_dir contains a :
|
||||
if container.bitcoin_mount_dir.find(":") == -1:
|
||||
container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir)
|
||||
del container.bitcoin_mount_dir
|
||||
|
||||
return app
|
||||
|
|
|
|||
171
app/lib/composegenerator/shared/networking.py
Normal file
171
app/lib/composegenerator/shared/networking.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import json
|
||||
from os import path
|
||||
import random
|
||||
from lib.composegenerator.shared.utils.networking import getContainerHiddenService
|
||||
from lib.composegenerator.v2.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig, App, Container
|
||||
from lib.citadelutils import parse_dotenv
|
||||
from dacite import from_dict
|
||||
|
||||
def getMainContainer(app: App) -> Container:
|
||||
if len(app.containers) == 1:
|
||||
return app.containers[0]
|
||||
else:
|
||||
for container in app.containers:
|
||||
# Main is recommended, support web for easier porting from Umbrel
|
||||
if container.name == 'main' or container.name == 'web':
|
||||
return container
|
||||
# Fallback to first container
|
||||
return app.containers[0]
|
||||
|
||||
def assignIpV4(appId: str, containerName: str):
|
||||
scriptDir = path.dirname(path.realpath(__file__))
|
||||
nodeRoot = path.join(scriptDir, "..", "..", "..", "..")
|
||||
networkingFile = path.join(nodeRoot, "apps", "networking.json")
|
||||
envFile = path.join(nodeRoot, ".env")
|
||||
cleanContainerName = containerName.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if cleanContainerName.find("\n") != -1:
|
||||
raise Exception("Newline in container name")
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
cleanContainerName.upper().replace("-", "_")
|
||||
)
|
||||
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
|
||||
# can be used
|
||||
usedIps = []
|
||||
networkingData = {}
|
||||
if path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
|
||||
if 'ip_addresses' in networkingData:
|
||||
usedIps = list(networkingData['ip_addresses'].values())
|
||||
else:
|
||||
networkingData['ip_addresses'] = {}
|
||||
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
|
||||
# If the IP is already in use, it will be tried again until it's not in use
|
||||
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
|
||||
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
|
||||
# Throw an error, because no more IPs can be used
|
||||
if len(usedIps) == 235:
|
||||
raise Exception("No more IPs can be used")
|
||||
|
||||
if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']:
|
||||
ip = networkingData['ip_addresses']["{}-{}".format(
|
||||
appId, cleanContainerName)]
|
||||
else:
|
||||
while True:
|
||||
ip = "10.21.21." + str(random.randint(20, 255))
|
||||
if ip not in usedIps:
|
||||
networkingData['ip_addresses']["{}-{}".format(
|
||||
appId, cleanContainerName)] = ip
|
||||
break
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
|
||||
return
|
||||
|
||||
with open(envFile, 'a') as f:
|
||||
f.write("{}={}\n".format(env_var, ip))
|
||||
with open(networkingFile, 'w') as f:
|
||||
json.dump(networkingData, f)
|
||||
|
||||
def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
|
||||
# Strip leading/trailing whitespace from container.name
|
||||
container.name = container.name.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if container.name.find("\n") != -1:
|
||||
raise Exception("Newline in container name")
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
|
||||
# can be used
|
||||
usedIps = []
|
||||
networkingData = {}
|
||||
if path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
|
||||
if 'ip_addresses' in networkingData:
|
||||
usedIps = list(networkingData['ip_addresses'].values())
|
||||
else:
|
||||
networkingData['ip_addresses'] = {}
|
||||
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
|
||||
# If the IP is already in use, it will be tried again until it's not in use
|
||||
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
|
||||
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
|
||||
# Throw an error, because no more IPs can be used
|
||||
if len(usedIps) == 235:
|
||||
raise Exception("No more IPs can be used")
|
||||
|
||||
if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
|
||||
ip = networkingData['ip_addresses']["{}-{}".format(
|
||||
appId, container.name)]
|
||||
else:
|
||||
while True:
|
||||
ip = "10.21.21." + str(random.randint(20, 255))
|
||||
if ip not in usedIps:
|
||||
networkingData['ip_addresses']["{}-{}".format(
|
||||
appId, container.name)] = ip
|
||||
break
|
||||
container.networks = from_dict(data_class=NetworkConfig, data={'default': {
|
||||
'ipv4_address': "$" + env_var}})
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
|
||||
return container
|
||||
|
||||
# Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile
|
||||
with open(envFile, 'a') as f:
|
||||
f.write("{}={}\n".format(env_var, ip))
|
||||
with open(networkingFile, 'w') as f:
|
||||
json.dump(networkingData, f)
|
||||
return container
|
||||
|
||||
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
|
||||
for container in app.containers:
|
||||
if container.network_mode and container.network_mode == "host":
|
||||
continue
|
||||
if container.noNetwork:
|
||||
# Check if port is defined for the container
|
||||
if container.port:
|
||||
raise Exception("Port defined for container without network")
|
||||
if getMainContainer(app).name == container.name:
|
||||
raise Exception("Main container without network")
|
||||
# Skip this iteration of the loop
|
||||
continue
|
||||
|
||||
container = assignIp(container, app.metadata.id,
|
||||
networkingFile, envFile)
|
||||
|
||||
return app
|
||||
|
||||
def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3:
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
hiddenServices = ""
|
||||
|
||||
mainContainer = getMainContainer(app)
|
||||
|
||||
for container in app.containers:
|
||||
if container.network_mode and container.network_mode == "host":
|
||||
continue
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
app.metadata.id.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
hiddenServices += getContainerHiddenService(
|
||||
app.metadata, container, dotEnv[env_var], container.name == mainContainer.name)
|
||||
if container.hiddenServicePorts:
|
||||
del container.hiddenServicePorts
|
||||
|
||||
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
|
||||
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
|
||||
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
|
||||
f.write(hiddenServices)
|
||||
return app
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from lib.composegenerator.v1.types import App, AppStage4, generateApp
|
||||
from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort
|
||||
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
|
||||
from lib.composegenerator.shared.env import validateEnv
|
||||
from lib.citadelutils import classToDict
|
||||
import os
|
||||
|
||||
def createComposeConfigFromV1(app: dict, nodeRoot: str):
|
||||
envFile = os.path.join(nodeRoot, ".env")
|
||||
networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
|
||||
|
||||
newApp: App = generateApp(app)
|
||||
newApp = convertContainerPermissions(newApp)
|
||||
validateEnv(newApp)
|
||||
newApp = convertDataDirToVolume(newApp)
|
||||
newApp = configureIps(newApp, networkingFile, envFile)
|
||||
newApp = configureMainPort(newApp, nodeRoot)
|
||||
configureHiddenServices(newApp, nodeRoot)
|
||||
finalConfig: AppStage4 = convertContainersToServices(newApp)
|
||||
newApp = classToDict(finalConfig)
|
||||
del newApp['metadata']
|
||||
if "version" in newApp:
|
||||
del newApp["version"]
|
||||
# Set version to 3.8 (current compose file version)
|
||||
newApp = {'version': '3.8', **newApp}
|
||||
return newApp
|
||||
|
|
@ -1,226 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from dacite import from_dict
|
||||
from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig
|
||||
from lib.citadelutils import parse_dotenv
|
||||
import json
|
||||
from os import path
|
||||
import random
|
||||
from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService
|
||||
|
||||
|
||||
def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
|
||||
# Strip leading/trailing whitespace from container.name
|
||||
container.name = container.name.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if container.name.find("\n") != -1:
|
||||
raise Exception("Newline in container name")
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
# Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
|
||||
# can be used
|
||||
usedIps = []
|
||||
networkingData = {}
|
||||
if path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
|
||||
if 'ip_addresses' in networkingData:
|
||||
usedIps = list(networkingData['ip_addresses'].values())
|
||||
else:
|
||||
networkingData['ip_addresses'] = {}
|
||||
# An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
|
||||
# If the IP is already in use, it will be tried again until it's not in use
|
||||
# If it's not in use, it will be added to the usedIps list and written to the usedIpFile
|
||||
# If the usedIpsFile contains all IPs between 10.21.21.20 and 10.21.21.255 (inclusive),
|
||||
# Throw an error, because no more IPs can be used
|
||||
if len(usedIps) == 235:
|
||||
raise Exception("No more IPs can be used")
|
||||
|
||||
if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
|
||||
ip = networkingData['ip_addresses']["{}-{}".format(
|
||||
appId, container.name)]
|
||||
else:
|
||||
while True:
|
||||
ip = "10.21.21." + str(random.randint(20, 255))
|
||||
if ip not in usedIps:
|
||||
networkingData['ip_addresses']["{}-{}".format(
|
||||
appId, container.name)] = ip
|
||||
break
|
||||
container.networks = from_dict(data_class=NetworkConfig, data={'default': {
|
||||
'ipv4_address': "$" + env_var}})
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
|
||||
return container
|
||||
|
||||
# Now append a new line with APP_{app_name}_{container_name}_IP=${IP} to the envFile
|
||||
with open(envFile, 'a') as f:
|
||||
f.write("{}={}\n".format(env_var, ip))
|
||||
with open(networkingFile, 'w') as f:
|
||||
json.dump(networkingData, f)
|
||||
return container
|
||||
|
||||
|
||||
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
|
||||
# Strip leading/trailing whitespace from container.name
|
||||
container.name = container.name.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
|
||||
raise Exception("Newline or space in container name")
|
||||
|
||||
env_var = "APP_{}_{}_PORT".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
|
||||
port = getFreePort(networkingFile, appId)
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
|
||||
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
|
||||
with open(envFile, 'a') as f:
|
||||
f.write("{}={}\n".format(env_var, port))
|
||||
|
||||
# This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
|
||||
# where the outer {{ }} will be replaced by {} in the returned string
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
|
||||
|
||||
def getMainContainer(app: dict):
|
||||
if len(app.containers) == 1:
|
||||
return app.containers[0]
|
||||
else:
|
||||
if not app.metadata.mainContainer:
|
||||
app.metadata.mainContainer = 'main'
|
||||
for container in app.containers:
|
||||
if container.name == app.metadata.mainContainer:
|
||||
return container
|
||||
raise Exception(
|
||||
"No main container found for app {}".format(app.metadata.name))
|
||||
|
||||
|
||||
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
|
||||
registryFile = path.join(nodeRoot, "apps", "registry.json")
|
||||
registry: list = []
|
||||
if path.isfile(registryFile):
|
||||
with open(registryFile, 'r') as f:
|
||||
registry = json.load(f)
|
||||
else:
|
||||
raise Exception("Registry file not found")
|
||||
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
|
||||
mainContainer = getMainContainer(app)
|
||||
|
||||
portDetails = assignPort(mainContainer, app.metadata.id, path.join(
|
||||
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
|
||||
containerPort = portDetails['port']
|
||||
portAsEnvVar = portDetails['env_var']
|
||||
portToAppend = portAsEnvVar
|
||||
|
||||
mainPort = False
|
||||
|
||||
if mainContainer.port:
|
||||
portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port)
|
||||
mainPort = mainContainer.port
|
||||
del mainContainer.port
|
||||
else:
|
||||
portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar)
|
||||
|
||||
if mainContainer.ports:
|
||||
mainContainer.ports.append(portToAppend)
|
||||
# Set the main port to the first port in the list, if it contains a :, it's the port after the :
|
||||
# If it doesn't contain a :, it's the port itself
|
||||
if mainPort == False:
|
||||
mainPort = mainContainer.ports[0]
|
||||
if mainPort.find(":") != -1:
|
||||
mainPort = mainPort.split(":")[1]
|
||||
else:
|
||||
mainContainer.ports = [portToAppend]
|
||||
if mainPort == False:
|
||||
mainPort = portDetails['port']
|
||||
|
||||
mainContainer = assignIp(mainContainer, app.metadata.id, path.join(
|
||||
nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
|
||||
|
||||
# If the IP wasn't in dotenv before, now it should be
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
|
||||
containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace(
|
||||
"-", "_"), mainContainer.name.upper().replace("-", "_"))]
|
||||
|
||||
hiddenservice = getHiddenService(
|
||||
app.metadata.name, app.metadata.id, containerIP, mainPort)
|
||||
|
||||
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
|
||||
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
|
||||
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
|
||||
f.write(hiddenservice)
|
||||
|
||||
# Also set the port in metadata
|
||||
app.metadata.port = int(containerPort)
|
||||
|
||||
for registryApp in registry:
|
||||
if registryApp['id'] == app.metadata.id:
|
||||
registry[registry.index(registryApp)]['port'] = int(containerPort)
|
||||
break
|
||||
|
||||
with open(registryFile, 'w') as f:
|
||||
json.dump(registry, f, indent=4, sort_keys=True)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
|
||||
for container in app.containers:
|
||||
if container.noNetwork:
|
||||
# Check if port is defined for the container
|
||||
if container.port:
|
||||
raise Exception("Port defined for container without network")
|
||||
if app.metadata.mainContainer == container.name:
|
||||
raise Exception("Main container without network")
|
||||
# Skip this iteration of the loop
|
||||
continue
|
||||
|
||||
container = assignIp(container, app.metadata.id,
|
||||
networkingFile, envFile)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def configureHiddenServices(app: dict, nodeRoot: str) -> None:
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
hiddenServices = ""
|
||||
|
||||
if len(app.containers) == 1:
|
||||
mainContainer = app.containers[0]
|
||||
else:
|
||||
mainContainer = None
|
||||
if app.metadata.mainContainer == None:
|
||||
app.metadata.mainContainer = 'main'
|
||||
for container in app.containers:
|
||||
if container.name == app.metadata.mainContainer:
|
||||
mainContainer = container
|
||||
break
|
||||
if mainContainer is None:
|
||||
raise Exception("No main container found")
|
||||
|
||||
for container in app.containers:
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
app.metadata.id.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
hiddenServices += getContainerHiddenService(
|
||||
app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name)
|
||||
|
||||
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
|
||||
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
|
||||
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
|
||||
f.write(hiddenServices)
|
||||
|
|
@ -1,151 +0,0 @@
|
|||
from typing import Union, List
|
||||
from dataclasses import dataclass, field
|
||||
from dacite import from_dict
|
||||
|
||||
@dataclass
|
||||
class Metadata:
|
||||
id: str
|
||||
name: str
|
||||
version: str
|
||||
category: str
|
||||
tagline: str
|
||||
description: str
|
||||
developer: str
|
||||
website: str
|
||||
repo: str
|
||||
support: str
|
||||
gallery: List[str] = field(default_factory=list)
|
||||
dependencies: List[str] = field(default_factory=list)
|
||||
mainContainer: Union[str, None] = None
|
||||
updateContainer: Union[str, None] = None
|
||||
path: str = ""
|
||||
defaultPassword: str = ""
|
||||
torOnly: bool = False
|
||||
|
||||
@dataclass
|
||||
class Container:
|
||||
name: str
|
||||
image: str
|
||||
permissions: list = field(default_factory=list)
|
||||
ports: list = field(default_factory=list)
|
||||
port: Union[int, None] = None
|
||||
environment: Union[dict, None] = None
|
||||
data: list = field(default_factory=list)
|
||||
user: Union[str, None] = None
|
||||
stop_grace_period: str = '1m'
|
||||
depends_on: list = field(default_factory=list)
|
||||
entrypoint: Union[List[str], str] = field(default_factory=list)
|
||||
bitcoin_mount_dir: Union[str, None] = None
|
||||
command: Union[List[str], str] = field(default_factory=list)
|
||||
init: Union[bool, None] = None
|
||||
stop_signal: Union[str, None] = None
|
||||
noNetwork: Union[bool, None] = None
|
||||
needsHiddenService: Union[bool, None] = None
|
||||
hiddenServicePort: Union[int, None] = None
|
||||
hiddenServicePorts: Union[dict, None] = None
|
||||
environment_allow: list = field(default_factory=list)
|
||||
# Only added later
|
||||
volumes: list = field(default_factory=list)
|
||||
restart: Union[str, None] = None
|
||||
|
||||
@dataclass
|
||||
class App:
|
||||
version: Union[str, int]
|
||||
metadata: Metadata
|
||||
containers: List[Container]
|
||||
|
||||
# Generate an app instance from an app dict
|
||||
def generateApp(appDict):
|
||||
return from_dict(data_class=App, data=appDict)
|
||||
|
||||
@dataclass
|
||||
class Network:
|
||||
ipv4_address: Union[str, None] = None
|
||||
|
||||
@dataclass
|
||||
class NetworkConfig:
|
||||
default: Network
|
||||
|
||||
# After converting data dir and defining volumes, stage 2
|
||||
@dataclass
|
||||
class ContainerStage2:
|
||||
id: str
|
||||
name: str
|
||||
image: str
|
||||
permissions: List[str] = field(default_factory=list)
|
||||
ports: list = field(default_factory=list)
|
||||
environment: Union[dict, None] = None
|
||||
user: Union[str, None] = None
|
||||
stop_grace_period: str = '1m'
|
||||
depends_on: List[str] = field(default_factory=list)
|
||||
entrypoint: Union[List[str], str] = field(default_factory=list)
|
||||
command: Union[List[str], str] = field(default_factory=list)
|
||||
init: Union[bool, None] = None
|
||||
stop_signal: Union[str, None] = None
|
||||
noNetwork: Union[bool, None] = None
|
||||
needsHiddenService: Union[bool, None] = None
|
||||
hiddenServicePort: Union[int, None] = None
|
||||
hiddenServicePorts: Union[dict, None] = None
|
||||
volumes: List[str] = field(default_factory=list)
|
||||
networks: NetworkConfig = field(default_factory=NetworkConfig)
|
||||
restart: Union[str, None] = None
|
||||
|
||||
@dataclass
|
||||
class AppStage2:
|
||||
version: Union[str, int]
|
||||
metadata: Metadata
|
||||
containers: List[ContainerStage2]
|
||||
|
||||
@dataclass
|
||||
class MetadataStage3:
|
||||
id: str
|
||||
name: str
|
||||
version: str
|
||||
category: str
|
||||
tagline: str
|
||||
description: str
|
||||
developer: str
|
||||
website: str
|
||||
dependencies: List[str]
|
||||
repo: str
|
||||
support: str
|
||||
gallery: List[str]
|
||||
mainContainer: Union[str, None] = None
|
||||
updateContainer: Union[str, None] = None
|
||||
path: str = ""
|
||||
defaultPassword: str = ""
|
||||
torOnly: bool = False
|
||||
|
||||
@dataclass
|
||||
class AppStage3:
|
||||
version: Union[str, int]
|
||||
metadata: MetadataStage3
|
||||
containers: List[ContainerStage2]
|
||||
|
||||
@dataclass
|
||||
class ContainerStage4:
|
||||
id: str
|
||||
name: str
|
||||
image: str
|
||||
ports: list = field(default_factory=list)
|
||||
environment: Union[dict, None] = None
|
||||
user: Union[str, None] = None
|
||||
stop_grace_period: str = '1m'
|
||||
depends_on: List[str] = field(default_factory=list)
|
||||
entrypoint: Union[List[str], str] = field(default_factory=list)
|
||||
command: Union[List[str], str] = field(default_factory=list)
|
||||
init: Union[bool, None] = None
|
||||
stop_signal: Union[str, None] = None
|
||||
noNetwork: Union[bool, None] = None
|
||||
needsHiddenService: Union[bool, None] = None
|
||||
hiddenServicePort: Union[int, None] = None
|
||||
hiddenServicePorts: Union[dict, None] = None
|
||||
volumes: List[str] = field(default_factory=list)
|
||||
networks: NetworkConfig = field(default_factory=NetworkConfig)
|
||||
restart: Union[str, None] = None
|
||||
|
||||
@dataclass
|
||||
class AppStage4:
|
||||
version: Union[str, int]
|
||||
metadata: MetadataStage3
|
||||
services: List[ContainerStage4]
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
|
||||
from lib.composegenerator.v1.types import Container
|
||||
|
||||
def getFreePort(networkingFile: str, appId: str):
|
||||
# Ports used currently in Citadel
|
||||
usedPorts = [
|
||||
# Dashboard
|
||||
80,
|
||||
# Sometimes used by nginx with some setups
|
||||
433,
|
||||
# Dashboard SSL
|
||||
443,
|
||||
# Bitcoin Core P2P
|
||||
8333,
|
||||
# LND gRPC
|
||||
10009,
|
||||
# LND REST
|
||||
8080,
|
||||
# Electrum Server
|
||||
50001,
|
||||
# Tor Proxy
|
||||
9050,
|
||||
]
|
||||
networkingData = {}
|
||||
if os.path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
if 'ports' in networkingData:
|
||||
usedPorts += list(networkingData['ports'].values())
|
||||
else:
|
||||
networkingData['ports'] = {}
|
||||
|
||||
if appId in networkingData['ports']:
|
||||
return networkingData['ports'][appId]
|
||||
|
||||
while True:
|
||||
port = str(random.randint(1024, 49151))
|
||||
if port not in usedPorts:
|
||||
# Check if anyhing is listening on the specific port
|
||||
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
|
||||
networkingData['ports'][appId] = port
|
||||
break
|
||||
|
||||
with open(networkingFile, 'w') as f:
|
||||
json.dump(networkingData, f)
|
||||
|
||||
return port
|
||||
|
||||
|
||||
def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str:
|
||||
hiddenServices = '''
|
||||
# {} Hidden Service
|
||||
HiddenServiceDir /var/lib/tor/app-{}
|
||||
'''.format(name, id)
|
||||
for port in ports:
|
||||
hiddenServices += 'HiddenServicePort {} {}:{}'.format(
|
||||
port, internalIp, port)
|
||||
hiddenServices += "\n"
|
||||
return hiddenServices
|
||||
|
||||
|
||||
def getHiddenServiceString(name: str, id: str, internalPort, internalIp: str, publicPort) -> str:
|
||||
return '''
|
||||
# {} Hidden Service
|
||||
HiddenServiceDir /var/lib/tor/app-{}
|
||||
HiddenServicePort {} {}:{}
|
||||
|
||||
'''.format(name, id, publicPort, internalIp, internalPort)
|
||||
|
||||
|
||||
def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str:
|
||||
return getHiddenServiceString(appName, appId, appPort, appIp, "80")
|
||||
|
||||
|
||||
def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str:
|
||||
if not container.needsHiddenService and not isMainContainer:
|
||||
return ""
|
||||
if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer:
|
||||
print("Container {} for app {} isn't compatible with hidden service assignment".format(
|
||||
container.name, appName))
|
||||
return ""
|
||||
|
||||
if isMainContainer:
|
||||
if not container.hiddenServicePorts:
|
||||
return ""
|
||||
# hiddenServicePorts is a map of hidden service name to port
|
||||
# We need to generate a hidden service for each one
|
||||
hiddenServices = ""
|
||||
for name, port in container.hiddenServicePorts.items():
|
||||
if ".." in name:
|
||||
print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName))
|
||||
|
||||
# If port is a list, use getHiddenServiceMultiPort
|
||||
if isinstance(port, list):
|
||||
hiddenServices += getHiddenServiceMultiPort("{} {}".format(appName, name), "{}-{}".format(
|
||||
appId, name), containerIp, port)
|
||||
else:
|
||||
hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format(
|
||||
appId, name), port, containerIp, port)
|
||||
del container.hiddenServicePorts
|
||||
return hiddenServices
|
||||
|
||||
del container.needsHiddenService
|
||||
if not container.port:
|
||||
data = getHiddenServiceString(appName + container.name, "{}-{}".format(
|
||||
appId, container.name), container.hiddenServicePort, containerIp, "80")
|
||||
del container.hiddenServicePort
|
||||
return data
|
||||
else:
|
||||
return getHiddenServiceString(appName + container.name, "{}-{}".format(
|
||||
appId, container.name), container.port, containerIp, container.port)
|
||||
|
|
@ -3,15 +3,34 @@
|
|||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from lib.composegenerator.v2.types import App, AppStage2, AppStage4, generateApp
|
||||
from lib.composegenerator.v2.networking import configureHiddenServices, configureIps, configureMainPort
|
||||
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
|
||||
from lib.composegenerator.v2.networking import configureMainPort
|
||||
from lib.composegenerator.shared.networking import configureHiddenServices, configureIps
|
||||
from lib.composegenerator.shared.main import convertContainerPermissions, convertContainersToServices
|
||||
from lib.composegenerator.shared.env import validateEnv
|
||||
from lib.citadelutils import classToDict
|
||||
import os
|
||||
|
||||
def convertDataDirToVolumeGen2(app: App) -> AppStage2:
|
||||
app = convertDataDirToVolume(app)
|
||||
for container in app.containers:
|
||||
# Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
|
||||
# Also, a datadir shouldn't start with a /
|
||||
for dataDir in container.data:
|
||||
if dataDir.find("..") == -1 and dataDir[0] != "/":
|
||||
container.volumes.append(
|
||||
'${APP_DATA_DIR}/' + dataDir)
|
||||
else:
|
||||
print("Data dir " + dataDir +
|
||||
" contains invalid characters")
|
||||
del container.data
|
||||
if container.bitcoin_mount_dir != None:
|
||||
if not 'bitcoind' in container.permissions:
|
||||
print("Warning: container {} of app {} defines bitcoin_mount_dir but has no permissions for bitcoind".format(container.name, app.metadata.name))
|
||||
# Skip this container
|
||||
continue
|
||||
# Also skip the container if container.bitcoin_mount_dir contains a :
|
||||
if container.bitcoin_mount_dir.find(":") == -1:
|
||||
container.volumes.append('${BITCOIN_DATA_DIR}:' + container.bitcoin_mount_dir)
|
||||
del container.bitcoin_mount_dir
|
||||
if container.lnd_mount_dir != None:
|
||||
if not 'lnd' in container.permissions:
|
||||
print("Warning: container {} of app {} defines lnd_mount_dir but doesn't request lnd permission".format(container.name, app.metadata.name))
|
||||
|
|
|
|||
|
|
@ -2,14 +2,59 @@
|
|||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container
|
||||
from lib.citadelutils import parse_dotenv
|
||||
from lib.composegenerator.v2.types import App, AppStage2, AppStage3, Container
|
||||
import json
|
||||
from os import path
|
||||
import os
|
||||
import random
|
||||
from lib.composegenerator.v2.utils.networking import getContainerHiddenService
|
||||
from lib.composegenerator.v1.networking import assignIp, assignPort
|
||||
from lib.composegenerator.shared.networking import assignIp
|
||||
from lib.citadelutils import FileLock
|
||||
|
||||
def getFreePort(networkingFile: str, appId: str):
|
||||
# Ports used currently in Citadel
|
||||
usedPorts = [
|
||||
# Dashboard
|
||||
80,
|
||||
# Sometimes used by nginx with some setups
|
||||
433,
|
||||
# Dashboard SSL
|
||||
443,
|
||||
# Bitcoin Core P2P
|
||||
8333,
|
||||
# LND gRPC
|
||||
10009,
|
||||
# LND REST
|
||||
8080,
|
||||
# Electrum Server
|
||||
50001,
|
||||
# Tor Proxy
|
||||
9050,
|
||||
]
|
||||
networkingData = {}
|
||||
if path.isfile(networkingFile):
|
||||
with open(networkingFile, 'r') as f:
|
||||
networkingData = json.load(f)
|
||||
if 'ports' in networkingData:
|
||||
usedPorts += list(networkingData['ports'].values())
|
||||
else:
|
||||
networkingData['ports'] = {}
|
||||
|
||||
if appId in networkingData['ports']:
|
||||
return networkingData['ports'][appId]
|
||||
|
||||
while True:
|
||||
port = str(random.randint(1024, 49151))
|
||||
if port not in usedPorts:
|
||||
# Check if anyhing is listening on the specific port
|
||||
if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
|
||||
networkingData['ports'][appId] = port
|
||||
break
|
||||
|
||||
with open(networkingFile, 'w') as f:
|
||||
json.dump(networkingData, f)
|
||||
|
||||
return port
|
||||
|
||||
def getMainContainer(app: App) -> Container:
|
||||
if len(app.containers) == 1:
|
||||
|
|
@ -22,8 +67,36 @@ def getMainContainer(app: App) -> Container:
|
|||
# Fallback to first container
|
||||
return app.containers[0]
|
||||
|
||||
def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
|
||||
# Strip leading/trailing whitespace from container.name
|
||||
container.name = container.name.strip()
|
||||
# If the name still contains a newline, throw an error
|
||||
if container.name.find("\n") != -1 or container.name.find(" ") != -1:
|
||||
raise Exception("Newline or space in container name")
|
||||
|
||||
env_var = "APP_{}_{}_PORT".format(
|
||||
appId.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
|
||||
port = getFreePort(networkingFile, appId)
|
||||
|
||||
dotEnv = parse_dotenv(envFile)
|
||||
if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
|
||||
# Now append a new line with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
|
||||
with open(envFile, 'a') as f:
|
||||
f.write("{}={}\n".format(env_var, port))
|
||||
|
||||
# This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
|
||||
# where the outer {{ }} will be replaced by {} in the returned string
|
||||
return {"port": port, "env_var": "${{{}}}".format(env_var)}
|
||||
|
||||
|
||||
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
|
||||
lock = FileLock("citadel_registry_lock", dir="/tmp")
|
||||
lock.acquire()
|
||||
registryFile = path.join(nodeRoot, "apps", "registry.json")
|
||||
registry: list = []
|
||||
if path.isfile(registryFile):
|
||||
|
|
@ -81,48 +154,5 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
|
|||
|
||||
with open(registryFile, 'w') as f:
|
||||
json.dump(registry, f, indent=4, sort_keys=True)
|
||||
|
||||
return app
|
||||
|
||||
def configureIps(app: AppStage2, networkingFile: str, envFile: str):
|
||||
for container in app.containers:
|
||||
if container.network_mode and container.network_mode == "host":
|
||||
continue
|
||||
if container.noNetwork:
|
||||
# Check if port is defined for the container
|
||||
if container.port:
|
||||
raise Exception("Port defined for container without network")
|
||||
if getMainContainer(app).name == container.name:
|
||||
raise Exception("Main container without network")
|
||||
# Skip this iteration of the loop
|
||||
continue
|
||||
|
||||
container = assignIp(container, app.metadata.id,
|
||||
networkingFile, envFile)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def configureHiddenServices(app: AppStage3, nodeRoot: str) -> AppStage3:
|
||||
dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
|
||||
hiddenServices = ""
|
||||
|
||||
mainContainer = getMainContainer(app)
|
||||
|
||||
for container in app.containers:
|
||||
if container.network_mode and container.network_mode == "host":
|
||||
continue
|
||||
env_var = "APP_{}_{}_IP".format(
|
||||
app.metadata.id.upper().replace("-", "_"),
|
||||
container.name.upper().replace("-", "_")
|
||||
)
|
||||
hiddenServices += getContainerHiddenService(
|
||||
app.metadata, container, dotEnv[env_var], container.name == mainContainer.name)
|
||||
if container.hiddenServicePorts:
|
||||
del container.hiddenServicePorts
|
||||
|
||||
torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
|
||||
torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
|
||||
with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
|
||||
f.write(hiddenServices)
|
||||
lock.release()
|
||||
return app
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@
|
|||
import os
|
||||
|
||||
from lib.citadelutils import classToDict
|
||||
from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainersToServices
|
||||
from lib.composegenerator.shared.main import convertContainersToServices
|
||||
from lib.composegenerator.shared.env import validateEnv
|
||||
from lib.composegenerator.v2.networking import configureIps, configureHiddenServices
|
||||
from lib.composegenerator.shared.networking import configureIps, configureHiddenServices
|
||||
|
||||
from lib.composegenerator.v3.types import App, AppStage2, AppStage4, generateApp
|
||||
from lib.composegenerator.v3.networking import configureMainPort
|
||||
|
|
|
|||
|
|
@ -2,13 +2,11 @@
|
|||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from lib.composegenerator.v3.types import App, AppStage2, AppStage3, Container
|
||||
from lib.citadelutils import parse_dotenv
|
||||
from lib.composegenerator.v3.types import App, AppStage2, AppStage3
|
||||
import json
|
||||
from os import path
|
||||
import random
|
||||
from lib.composegenerator.v1.networking import assignIp, assignPort
|
||||
|
||||
from lib.composegenerator.shared.networking import assignIp
|
||||
from lib.citadelutils import FileLock
|
||||
|
||||
def getMainContainerIndex(app: App):
|
||||
if len(app.containers) == 1:
|
||||
|
|
@ -27,6 +25,8 @@ def getMainContainerIndex(app: App):
|
|||
|
||||
|
||||
def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
|
||||
lock = FileLock("citadel_registry_lock", dir="/tmp")
|
||||
lock.acquire()
|
||||
registryFile = path.join(nodeRoot, "apps", "registry.json")
|
||||
portsFile = path.join(nodeRoot, "apps", "ports.json")
|
||||
envFile = path.join(nodeRoot, ".env")
|
||||
|
|
@ -101,8 +101,8 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
|
|||
|
||||
with open(registryFile, 'w') as f:
|
||||
json.dump(registry, f, indent=4, sort_keys=True)
|
||||
lock.release()
|
||||
|
||||
with open(envFile, 'a') as f:
|
||||
f.write("{}={}\n".format(portAsEnvVar, app.metadata.port))
|
||||
return app
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import random
|
|||
from typing import List
|
||||
from sys import argv
|
||||
import os
|
||||
import fcntl
|
||||
import requests
|
||||
import shutil
|
||||
import json
|
||||
|
|
@ -28,36 +27,12 @@ except Exception:
|
|||
print("Continuing anyway, but some features won't be available,")
|
||||
print("for example checking for app updates")
|
||||
|
||||
from lib.composegenerator.v1.generate import createComposeConfigFromV1
|
||||
from lib.composegenerator.v2.generate import createComposeConfigFromV2
|
||||
from lib.composegenerator.v3.generate import createComposeConfigFromV3
|
||||
from lib.validate import findAndValidateApps
|
||||
from lib.metadata import getAppRegistry
|
||||
from lib.entropy import deriveEntropy
|
||||
|
||||
class FileLock:
|
||||
"""Implements a file-based lock using flock(2).
|
||||
The lock file is saved in directory dir with name lock_name.
|
||||
dir is the current directory by default.
|
||||
"""
|
||||
|
||||
def __init__(self, lock_name, dir="."):
|
||||
self.lock_file = open(os.path.join(dir, lock_name), "w")
|
||||
|
||||
def acquire(self, blocking=True):
|
||||
"""Acquire the lock.
|
||||
If the lock is not already acquired, return None. If the lock is
|
||||
acquired and blocking is True, block until the lock is released. If
|
||||
the lock is acquired and blocking is False, raise an IOError.
|
||||
"""
|
||||
ops = fcntl.LOCK_EX
|
||||
if not blocking:
|
||||
ops |= fcntl.LOCK_NB
|
||||
fcntl.flock(self.lock_file, ops)
|
||||
|
||||
def release(self):
|
||||
"""Release the lock. Return None even if lock not currently acquired"""
|
||||
fcntl.flock(self.lock_file, fcntl.LOCK_UN)
|
||||
from lib.citadelutils import FileLock
|
||||
|
||||
# For an array of threads, join them and wait for them to finish
|
||||
def joinThreads(threads: List[threading.Thread]):
|
||||
|
|
@ -111,7 +86,7 @@ def handleAppV4(app):
|
|||
|
||||
for registryApp in registry:
|
||||
if registryApp['id'] == app:
|
||||
registry[registry.index(registryApp)]['port'] = resultYml["port"]
|
||||
registry[registry.index(registryApp)]['port'] = mainPort
|
||||
break
|
||||
|
||||
with open(registryFile, 'w') as f:
|
||||
|
|
@ -138,12 +113,19 @@ def getAppYml(name):
|
|||
|
||||
def update(verbose: bool = False):
|
||||
apps = findAndValidateApps(appsDir)
|
||||
portCache = {}
|
||||
try:
|
||||
with open(os.path.join(appsDir, "ports.cache.json"), "w") as f:
|
||||
portCache = json.load(f)
|
||||
except Exception: pass
|
||||
# The compose generation process updates the registry, so we need to get it set up with the basics before that
|
||||
registry = getAppRegistry(apps, appsDir)
|
||||
registry = getAppRegistry(apps, appsDir, portCache)
|
||||
with open(os.path.join(appsDir, "registry.json"), "w") as f:
|
||||
json.dump(registry["metadata"], f, sort_keys=True)
|
||||
with open(os.path.join(appsDir, "ports.json"), "w") as f:
|
||||
json.dump(registry["ports"], f, sort_keys=True)
|
||||
with open(os.path.join(appsDir, "ports.cache.json"), "w") as f:
|
||||
json.dump(registry["portCache"], f, sort_keys=True)
|
||||
with open(os.path.join(appsDir, "virtual-apps.json"), "w") as f:
|
||||
json.dump(registry["virtual_apps"], f, sort_keys=True)
|
||||
print("Wrote registry to registry.json")
|
||||
|
|
@ -234,17 +216,14 @@ def stopInstalled():
|
|||
# Loads an app.yml and converts it to a docker-compose.yml
|
||||
def getApp(app, appId: str):
|
||||
if not "metadata" in app:
|
||||
raise Exception("Error: Could not find metadata in " + appFile)
|
||||
raise Exception("Error: Could not find metadata in " + appId)
|
||||
app["metadata"]["id"] = appId
|
||||
|
||||
if 'version' in app and str(app['version']) == "1":
|
||||
print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId))
|
||||
return createComposeConfigFromV1(app, nodeRoot)
|
||||
elif 'version' in app and str(app['version']) == "2":
|
||||
print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId))
|
||||
if 'version' in app and str(app['version']) == "2":
|
||||
print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId))
|
||||
return createComposeConfigFromV2(app, nodeRoot)
|
||||
elif 'version' in app and str(app['version']) == "3":
|
||||
print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.3.0".format(appId))
|
||||
print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.1.5".format(appId))
|
||||
return createComposeConfigFromV3(app, nodeRoot)
|
||||
else:
|
||||
raise Exception("Error: Unsupported version of app.yml")
|
||||
|
|
|
|||
|
|
@ -6,12 +6,8 @@ import os
|
|||
import yaml
|
||||
import traceback
|
||||
|
||||
from lib.composegenerator.v2.networking import getMainContainer
|
||||
from lib.composegenerator.shared.networking import assignIpV4
|
||||
from lib.entropy import deriveEntropy
|
||||
from typing import List
|
||||
import json
|
||||
import random
|
||||
|
||||
appPorts = {}
|
||||
appPortMap = {}
|
||||
|
|
@ -37,9 +33,10 @@ def appPortsToMap():
|
|||
# Also check the path and defaultPassword and set them to an empty string if they don't exist
|
||||
# In addition, set id on the metadata to the name of the app
|
||||
# Return a list of all app's metadata
|
||||
def getAppRegistry(apps, app_path):
|
||||
def getAppRegistry(apps, app_path, portCache):
|
||||
app_metadata = []
|
||||
virtual_apps = {}
|
||||
appPorts = portCache
|
||||
for app in apps:
|
||||
app_yml_path = os.path.join(app_path, app, 'app.yml')
|
||||
if os.path.isfile(app_yml_path):
|
||||
|
|
@ -78,7 +75,8 @@ def getAppRegistry(apps, app_path):
|
|||
return {
|
||||
"virtual_apps": virtual_apps,
|
||||
"metadata": app_metadata,
|
||||
"ports": appPortMap
|
||||
"ports": appPortMap,
|
||||
"portCache": appPorts,
|
||||
}
|
||||
|
||||
citadelPorts = [
|
||||
|
|
@ -102,11 +100,11 @@ citadelPorts = [
|
|||
|
||||
lastPort = 3000
|
||||
|
||||
def getNewPort(appPorts, appId):
|
||||
def getNewPort(usedPorts, appId, containerName, allowExisting):
|
||||
lastPort2 = lastPort
|
||||
while lastPort2 in appPorts.keys() or lastPort2 in citadelPorts:
|
||||
if lastPort2 in appPorts.keys() and appPorts[lastPort2]["app"] == appId:
|
||||
return lastPort2
|
||||
while lastPort2 in usedPorts.keys() or lastPort2 in citadelPorts:
|
||||
if allowExisting and lastPort2 in usedPorts.keys() and usedPorts[lastPort2]["app"] == appId and usedPorts[lastPort2]["container"] == containerName:
|
||||
break
|
||||
lastPort2 = lastPort2 + 1
|
||||
return lastPort2
|
||||
|
||||
|
|
@ -120,10 +118,10 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna
|
|||
"dynamic": isDynamic,
|
||||
}
|
||||
else:
|
||||
if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != appContainer["name"]:
|
||||
newPort = getNewPort(appPorts, appId)
|
||||
if port in citadelPorts or appPorts[port]["app"] != appId or appPorts[port]["container"] != containerName:
|
||||
if port in appPorts and priority > appPorts[port]["priority"]:
|
||||
#print("Prioritizing app {} over {}".format(appId, appPorts[port]["app"]))
|
||||
newPort = getNewPort(appPorts, appPorts[port]["app"], appPorts[port]["container"], False)
|
||||
appPorts[newPort] = appPorts[port].copy()
|
||||
appPorts[port] = {
|
||||
"app": appId,
|
||||
|
|
@ -137,7 +135,8 @@ def validatePort(containerName, appContainer, port, appId, priority: int, isDyna
|
|||
disabledApps.append(appId)
|
||||
print("App {} disabled because of port conflict".format(appId))
|
||||
else:
|
||||
#print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, appContainer["name"], newPort, port))
|
||||
newPort = getNewPort(appPorts, appId, containerName, True)
|
||||
#print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, containerName, newPort, port))
|
||||
appPorts[newPort] = {
|
||||
"app": appId,
|
||||
"port": port,
|
||||
|
|
@ -167,7 +166,8 @@ def getPortsV3App(app, appId):
|
|||
else:
|
||||
validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0)
|
||||
elif "requiredPorts" not in appContainer and "requiredUdpPorts" not in appContainer:
|
||||
validatePort(appContainer["name"], appContainer, getNewPort(appPorts, appId), appId, 0, True)
|
||||
# if the container does not define a port, assume 3000, and pass it to the container as env var
|
||||
validatePort(appContainer["name"], appContainer, 3000, appId, 0, True)
|
||||
if "requiredPorts" in appContainer:
|
||||
for port in appContainer["requiredPorts"]:
|
||||
validatePort(appContainer["name"], appContainer, port, appId, 2)
|
||||
|
|
|
|||
|
|
@ -11,8 +11,6 @@ import traceback
|
|||
scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
|
||||
nodeRoot = os.path.join(scriptDir, "..")
|
||||
|
||||
with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f:
|
||||
schemaVersion1 = yaml.safe_load(f)
|
||||
with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f:
|
||||
schemaVersion2 = yaml.safe_load(f)
|
||||
with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f:
|
||||
|
|
@ -24,15 +22,7 @@ with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file:
|
|||
# Validates app data
|
||||
# Returns true if valid, false otherwise
|
||||
def validateApp(app: dict):
|
||||
if 'version' in app and str(app['version']) == "1":
|
||||
try:
|
||||
validate(app, schemaVersion1)
|
||||
return True
|
||||
# Catch and log any errors, and return false
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
elif 'version' in app and str(app['version']) == "2":
|
||||
if 'version' in app and str(app['version']) == "2":
|
||||
try:
|
||||
validate(app, schemaVersion2)
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
compose: v2.10.0
|
||||
compose: v2.10.2
|
||||
dashboard: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e
|
||||
manager: ghcr.io/runcitadel/manager:deno@sha256:38ef8474cc501d3f3e9ea63e73d1c48f848662467ffe5f7f0b9bbb44e04055cf
|
||||
middleware: ghcr.io/runcitadel/middleware:main@sha256:2aa20f31001ab9e61cda548acbd1864a598728731ad6121f050c6a41503866ae
|
||||
|
|
|
|||
|
|
@ -99,7 +99,6 @@ services:
|
|||
default:
|
||||
ipv4_address: $LND_IP
|
||||
dashboard:
|
||||
container_name: dashboard
|
||||
image: ghcr.io/runcitadel/dashboard:v0.0.17@sha256:4416254a023b3060338529446068b97b2d95834c59119b75bdeae598c5c81d0e
|
||||
restart: on-failure
|
||||
stop_grace_period: 1m30s
|
||||
|
|
@ -231,6 +230,7 @@ services:
|
|||
networks:
|
||||
default:
|
||||
ipv4_address: $REDIS_IP
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: citadel_main_network
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user