Citadel 0.1.0 Preview 1

Implement quick updates (#56 )
Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com> Co-authored-by: Philipp Walter <philippwalter@pm.me>
2022-07-17 06:54:58 +00:00 · 2022-07-16 19:28:39 +02:00 · 2022-07-16 11:54:22 -05:00 · 2022-07-15 18:34:49 +02:00 · 2022-07-15 13:29:22 +02:00 · 2022-07-14 20:35:59 -04:00
31 changed files with 1264 additions and 1101 deletions
--- a/app-standard-v1.json
+++ b/app-standard-v1.json
@ -1,230 +0,0 @@
 {
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "Citadel app.yml v1",
  "description": "The first draft of Citadel's app.yml format",
  "type": "object",
  "properties": {
    "version": {
      "type": [
        "string",
        "number"
      ],
      "description": "The version of the app.yml format you're using."
    },
    "metadata": {
      "type": "object",
      "properties": {
        "name": {
          "description": "Displayed name of the app",
          "type": "string"
        },
        "version": {
          "description": "Displayed version for the app",
          "type": "string"
        },
        "category": {
          "description": "The category you'd put the app in",
          "type": "string"
        },
        "tagline": {
          "description": "A clever tagline",
          "type": "string"
        },
        "description": {
          "description": "A longer description of the app",
          "type": "string"
        },
        "developer": {
          "description": "The awesome people behind the app",
          "type": "string"
        },
        "website": {
          "description": "Displayed version for the app",
          "type": "string"
        },
        "dependencies": {
          "description": "The services the app depends on",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "repo": {
          "description": "The development repository for your app",
          "type": "string"
        },
        "support": {
          "description": "A link to the app support wiki/chat/...",
          "type": "string"
        },
        "gallery": {
          "type": "array",
          "description": "URLs or paths in the runcitadel/app-images/[app-name] folder with app images",
          "items": {
            "type": "string"
          }
        },
        "path": {
          "description": "The path of the app's visible site the open button should open",
          "type": "string"
        },
        "defaultPassword": {
          "description": "The app's default password",
          "type": "string"
        },
        "torOnly": {
          "description": "Whether the app is only available over tor",
          "type": "boolean"
        },
        "mainContainer": {
          "type": "string",
          "description": "The name of the main container for the app. If set, IP, port, and hidden service will be assigned to it automatically."
        },
        "updateContainer": {
          "type": "string",
          "description": "The container the developer system should automatically update."
        }
      },
      "required": [
        "name",
        "version",
        "category",
        "tagline",
        "description",
        "developer",
        "website",
        "repo",
        "support",
        "gallery"
      ],
      "additionalProperties": false
    },
    "containers": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "name": {
            "type": "string"
          },
          "image": {
            "type": "string"
          },
          "permissions": {
            "type": "array",
            "items": {
              "type": "string",
              "enum": [
                "lnd",
                "bitcoind",
                "electrum",
                "root",
                "hw"
              ]
            }
          },
          "ports": {
            "type": "array",
            "items": {
              "type": [
                "string",
                "number"
              ]
            }
          },
          "port": {
            "type": "number",
            "description": "If this is the main container, the port inside the container which will be exposed to the outside as the port specified in metadata."
          },
          "environment": {
            "type": "object"
          },
          "data": {
            "type": "array",
            "description": "An array of at directories in the container the app stores its data in. Can be empty. Please only list top-level directories.",
            "items": {
              "type": "string"
            }
          },
          "user": {
            "type": "string",
            "description": "The user the container should run as"
          },
          "stop_grace_period": {
            "type": "string",
            "description": "The grace period for stopping the container. Defaults to 1 minute."
          },
          "depends_on": {
            "type": "array",
            "description": "The services the container depends on"
          },
          "entrypoint": {
            "type": [
              "string",
              "array"
            ],
            "description": "The entrypoint for the container"
          },
          "bitcoin_mount_dir": {
            "type": "string",
            "description": "Where to mount the bitcoin dir"
          },
          "command": {
            "type": [
              "string",
              "array"
            ],
            "description": "The command for the container"
          },
          "init": {
            "type": "boolean",
            "description": "Whether the container should be run with init"
          },
          "stop_signal": {
            "type": "string",
            "description": "The signal to send to the container when stopping"
          },
          "noNetwork": {
            "type": "boolean",
            "description": "Set this to true if the container shouldn't get an IP & port exposed."
          },
          "needsHiddenService": {
            "type": "boolean",
            "description": "Set this to true if the container should be assigned a hidden service even if it's not the main container."
          },
          "hiddenServicePort": {
            "type": "number",
            "description": "Set this to a port if your container exposes multiple ports, but only one should be a hidden service."
          },
          "hiddenServicePorts": {
            "type": "object",
            "description": "Set this to a map of service names to hidden service ports if your container exposes multiple ports, and all of them should be hidden services.",
            "patternProperties": {
              "^[a-zA-Z0-9_]+$": {
                "type": [
                  "number",
                  "array"
                ]
              }
            }
          },
          "restart": {
            "type": "string",
            "description": "When the container should restart. Can be 'always' or 'on-failure'."
          }
        },
        "additionalProperties": false,
        "required": [
          "name",
          "image"
        ]
      },
      "additionalProperties": false
    }
  },
  "required": [
    "metadata",
    "containers"
  ],
  "additionalProperties": false
 }
--- a/app-standard-v1.json.license
+++ b/app-standard-v1.json.license
@ -1,3 +0,0 @@
 SPDX-FileCopyrightText: 2021 Citadel and contributors
 SPDX-License-Identifier: GPL-3.0-or-later
--- a/app-system/sources.list
+++ b/app-system/sources.list
@ -3,9 +3,9 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 # A collection of fully FLOSS app definitions and FLOSS apps for Citadel.
-https://github.com/runcitadel/apps v3-stable
+https://github.com/runcitadel/apps v4-beta
 # Some apps modified version of Umbrel apps, and their app definitions aren't FLOSS yet.
 # Include them anyway, but as a separate repo.
 # Add a # to the line below to disable the repo and only use FLOSS apps.
-https://github.com/runcitadel/apps-nonfree v3-stable
+https://github.com/runcitadel/apps-nonfree v3-beta
--- a/app/app-standard-v1.yml
+++ b/app/app-standard-v1.yml
@ -1,194 +0,0 @@
 # yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema
 $schema: https://json-schema.org/draft/2020-12/schema
 title: Citadel app.yml v1
 description: The first draft of Citadel's app.yml format
 type: object
 properties:
  version:
    type:
      - string
      - number
    description: The version of the app.yml format you're using.
  metadata:
    type: object
    properties:
      name:
        description: Displayed name of the app
        type: string
      version:
        description: Displayed version for the app
        type: string
      category:
        description: The category you'd put the app in
        type: string
      tagline:
        description: A clever tagline
        type: string
      description:
        description: A longer description of the app
        type: string
      developer:
        description: The awesome people behind the app
        type: string
      website:
        description: Displayed version for the app
        type: string
      dependencies:
        description: The services the app depends on
        type: array
        items:
          type: string
      repo:
        description: The development repository for your app
        type: string
      support:
        description: A link to the app support wiki/chat/...
        type: string
      gallery:
        type: array
        description: >-
          URLs or paths in the runcitadel/app-images/[app-name] folder with app
          images
        items:
          type: string
      path:
        description: The path of the app's visible site the open button should open
        type: string
      defaultPassword:
        description: The app's default password
        type: string
      torOnly:
        description: Whether the app is only available over tor
        type: boolean
      mainContainer:
        type: string
        description: >-
          The name of the main container for the app. If set, IP, port, and
          hidden service will be assigned to it automatically.
      updateContainer:
        type: string
        description: The container the developer system should automatically update.
    required:
      - name
      - version
      - category
      - tagline
      - description
      - developer
      - website
      - repo
      - support
      - gallery
    additionalProperties: false
  containers:
    type: array
    items:
      type: object
      properties:
        name:
          type: string
        image:
          type: string
        permissions:
          type: array
          items:
            type: string
            enum:
              - lnd
              - bitcoind
              - electrum
              - root
              - hw
        ports:
          type: array
          items:
            type:
              - string
              - number
        port:
          type: number
          description: >-
            If this is the main container, the port inside the container which
            will be exposed to the outside as the port specified in metadata.
        environment:
          type: object
        data:
          type: array
          description: >-
            An array of at directories in the container the app stores its data
            in. Can be empty. Please only list top-level directories.
          items:
            type: string
        user:
          type: string
          description: The user the container should run as
        stop_grace_period:
          type: string
          description: The grace period for stopping the container. Defaults to 1 minute.
        depends_on:
          type: array
          description: The services the container depends on
        entrypoint:
          type:
            - string
            - array
          description: The entrypoint for the container
        bitcoin_mount_dir:
          type: string
          description: Where to mount the bitcoin dir
        command:
          type:
            - string
            - array
          description: The command for the container
        init:
          type: boolean
          description: Whether the container should be run with init
        stop_signal:
          type: string
          description: The signal to send to the container when stopping
        noNetwork:
          type: boolean
          description: >-
            Set this to true if the container shouldn't get an IP & port
            exposed.
        needsHiddenService:
          type: boolean
          description: >-
            Set this to true if the container should be assigned a hidden
            service even if it's not the main container.
        hiddenServicePort:
          type: number
          description: >-
            Set this to a port if your container exposes multiple ports, but
            only one should be a hidden service.
        hiddenServicePorts:
          type: object
          description: >-
            Set this to a map of service names to hidden service ports if your
            container exposes multiple ports, and all of them should be hidden
            services.
          patternProperties:
            ^[a-zA-Z0-9_]+$:
              type:
                - number
                - array
        restart:
          type: string
          description: When the container should restart. Can be 'always' or 'on-failure'.
      additionalProperties: false
      required:
        - name
        - image
    additionalProperties: false
 required:
  - metadata
  - containers
 additionalProperties: false
--- a/app/lib/composegenerator/next/stage1.py
+++ b/app/lib/composegenerator/next/stage1.py
@ -1,24 +0,0 @@
 from lib.citadelutils import classToDict
 from lib.composegenerator.shared.env import validateEnv
 from lib.composegenerator.v3.types import App, generateApp
 from lib.composegenerator.v3.generate import convertContainerPermissions
 def createCleanConfigFromV3(app: dict, nodeRoot: str):
    parsedApp: App = generateApp(app)
    for container in range(len(parsedApp.containers)):
        # TODO: Make this dynamic and not hardcoded
        if parsedApp.containers[container].requires and "c-lightning" in parsedApp.containers[container].requires:
            parsedApp.containers[container] = None
    parsedApp = convertContainerPermissions(parsedApp)
    parsedApp = validateEnv(parsedApp)
    finalApp = classToDict(parsedApp)
    try:
        finalApp['permissions'] = finalApp['metadata']['dependencies']
    except:
        finalApp['permissions'] = []
    finalApp['id'] = finalApp['metadata']['id']
    del finalApp['metadata']
    # Set version of the cache file format
    finalApp['version'] = "1"
    return finalApp
--- a/app/lib/composegenerator/shared/env.py
+++ b/app/lib/composegenerator/shared/env.py
@ -4,7 +4,7 @@
 import re
 from typing import Union
-from lib.composegenerator.v1.types import App
+from lib.composegenerator.v2.types import App
 from lib.composegenerator.shared.const import always_allowed_env
 from lib.citadelutils import checkArrayContainsAllElements, getEnvVars
--- a/app/lib/composegenerator/shared/main.py
+++ b/app/lib/composegenerator/shared/main.py
@ -3,7 +3,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 # Main functions
-from lib.composegenerator.v1.types import App, AppStage3, AppStage2, Container
+from lib.composegenerator.v2.types import App, AppStage3, AppStage2, Container
 from lib.composegenerator.shared.const import permissions
--- a/app/lib/composegenerator/shared/networking.py
+++ b/app/lib/composegenerator/shared/networking.py
@ -0,0 +1,190 @@
 # SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 import json
 from os import path
 import random
 from lib.composegenerator.v2.types import ContainerStage2, NetworkConfig
 from lib.citadelutils import parse_dotenv
 from dacite import from_dict
 def getFreePort(networkingFile: str, appId: str):
    # Ports used currently in Citadel
    usedPorts = [
        # Dashboard
        80,
        # Sometimes used by nginx with some setups
        433,
        # Dashboard SSL
        443,
        # Bitcoin Core P2P
        8333,
        # LND gRPC
        10009,
        # LND REST
        8080,
        # Electrum Server
        50001,
        # Tor Proxy
        9050,
    ]
    networkingData = {}
    if path.isfile(networkingFile):
        with open(networkingFile, 'r') as f:
            networkingData = json.load(f)
    if 'ports' in networkingData:
        usedPorts += list(networkingData['ports'].values())
    else:
        networkingData['ports'] = {}
    if appId in networkingData['ports']:
        return networkingData['ports'][appId]
    while True:
        port = str(random.randint(1024, 49151))
        if port not in usedPorts:
            # Check if anyhing is listening on the specific port
            if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
                networkingData['ports'][appId] = port
                break
    with open(networkingFile, 'w') as f:
        json.dump(networkingData, f)
    return port
 def assignIpV4(appId: str, containerName: str):
    scriptDir = path.dirname(path.realpath(__file__))
    nodeRoot = path.join(scriptDir, "..", "..", "..", "..")
    networkingFile = path.join(nodeRoot, "apps", "networking.json")
    envFile = path.join(nodeRoot, ".env")
    cleanContainerName = containerName.strip()
    # If the name still contains a newline, throw an error
    if cleanContainerName.find("\n") != -1:
        raise Exception("Newline in container name")
    env_var = "APP_{}_{}_IP".format(
        appId.upper().replace("-", "_"),
        cleanContainerName.upper().replace("-", "_")
    )
    # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
    # can be used
    usedIps = []
    networkingData = {}
    if path.isfile(networkingFile):
        with open(networkingFile, 'r') as f:
            networkingData = json.load(f)
    if 'ip_addresses' in networkingData:
        usedIps = list(networkingData['ip_addresses'].values())
    else:
        networkingData['ip_addresses'] = {}
    # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
    # If the IP is already in use, it will be tried again until it's not in use
    # If it's not in use, it will be added to the usedIps list and written to the usedIpFile
    # If the usedIpsFile contains all IPs between  10.21.21.20 and  10.21.21.255 (inclusive),
    # Throw an error, because no more IPs can be used
    if len(usedIps) == 235:
        raise Exception("No more IPs can be used")
    if "{}-{}".format(appId, cleanContainerName) in networkingData['ip_addresses']:
        ip = networkingData['ip_addresses']["{}-{}".format(
            appId, cleanContainerName)]
    else:
        while True:
            ip = "10.21.21." + str(random.randint(20, 255))
            if ip not in usedIps:
                networkingData['ip_addresses']["{}-{}".format(
                    appId, cleanContainerName)] = ip
                break
    dotEnv = parse_dotenv(envFile)
    if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
        return
    with open(envFile, 'a') as f:
        f.write("{}={}\n".format(env_var, ip))
    with open(networkingFile, 'w') as f:
        json.dump(networkingData, f)
 def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
    # Strip leading/trailing whitespace from container.name
    container.name = container.name.strip()
    # If the name still contains a newline, throw an error
    if container.name.find("\n") != -1:
        raise Exception("Newline in container name")
    env_var = "APP_{}_{}_IP".format(
        appId.upper().replace("-", "_"),
        container.name.upper().replace("-", "_")
    )
    # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
    # can be used
    usedIps = []
    networkingData = {}
    if path.isfile(networkingFile):
        with open(networkingFile, 'r') as f:
            networkingData = json.load(f)
    if 'ip_addresses' in networkingData:
        usedIps = list(networkingData['ip_addresses'].values())
    else:
        networkingData['ip_addresses'] = {}
    # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
    # If the IP is already in use, it will be tried again until it's not in use
    # If it's not in use, it will be added to the usedIps list and written to the usedIpFile
    # If the usedIpsFile contains all IPs between  10.21.21.20 and  10.21.21.255 (inclusive),
    # Throw an error, because no more IPs can be used
    if len(usedIps) == 235:
        raise Exception("No more IPs can be used")
    if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
        ip = networkingData['ip_addresses']["{}-{}".format(
            appId, container.name)]
    else:
        while True:
            ip = "10.21.21." + str(random.randint(20, 255))
            if ip not in usedIps:
                networkingData['ip_addresses']["{}-{}".format(
                    appId, container.name)] = ip
                break
    container.networks = from_dict(data_class=NetworkConfig, data={'default': {
        'ipv4_address': "$" + env_var}})
    dotEnv = parse_dotenv(envFile)
    if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
        return container
    # Now append a new line  with APP_{app_name}_{container_name}_IP=${IP} to the envFile
    with open(envFile, 'a') as f:
        f.write("{}={}\n".format(env_var, ip))
    with open(networkingFile, 'w') as f:
        json.dump(networkingData, f)
    return container
 def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
    # Strip leading/trailing whitespace from container.name
    container.name = container.name.strip()
    # If the name still contains a newline, throw an error
    if container.name.find("\n") != -1 or container.name.find(" ") != -1:
        raise Exception("Newline or space in container name")
    env_var = "APP_{}_{}_PORT".format(
        appId.upper().replace("-", "_"),
        container.name.upper().replace("-", "_")
    )
    port = getFreePort(networkingFile, appId)
    dotEnv = parse_dotenv(envFile)
    if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
        return {"port": port, "env_var": "${{{}}}".format(env_var)}
    # Now append a new line  with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
    with open(envFile, 'a') as f:
        f.write("{}={}\n".format(env_var, port))
    # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
    # where the outer {{ }} will be replaced by {} in the returned string
    return {"port": port, "env_var": "${{{}}}".format(env_var)}
--- a/app/lib/composegenerator/v1/generate.py
+++ b/app/lib/composegenerator/v1/generate.py
@ -1,30 +0,0 @@
 # SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 from lib.composegenerator.v1.types import App, AppStage4, generateApp
 from lib.composegenerator.v1.networking import configureHiddenServices, configureIps, configureMainPort
 from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainerPermissions, convertContainersToServices
 from lib.composegenerator.shared.env import validateEnv
 from lib.citadelutils import classToDict
 import os
 def createComposeConfigFromV1(app: dict, nodeRoot: str):
    envFile = os.path.join(nodeRoot, ".env")
    networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
    newApp: App = generateApp(app)
    newApp = convertContainerPermissions(newApp)
    validateEnv(newApp)
    newApp = convertDataDirToVolume(newApp)
    newApp = configureIps(newApp, networkingFile, envFile)
    newApp = configureMainPort(newApp, nodeRoot)
    configureHiddenServices(newApp, nodeRoot)
    finalConfig: AppStage4 = convertContainersToServices(newApp)
    newApp = classToDict(finalConfig)
    del newApp['metadata']
    if "version" in newApp:
        del newApp["version"]
    # Set version to 3.8 (current compose file version)
    newApp = {'version': '3.8', **newApp}
    return newApp
--- a/app/lib/composegenerator/v1/networking.py
+++ b/app/lib/composegenerator/v1/networking.py
@ -1,226 +0,0 @@
 # SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 from dacite import from_dict
 from lib.composegenerator.v1.types import AppStage2, AppStage3, ContainerStage2, NetworkConfig
 from lib.citadelutils import parse_dotenv
 import json
 from os import path
 import random
 from lib.composegenerator.v1.utils.networking import getContainerHiddenService, getFreePort, getHiddenService
 def assignIp(container: ContainerStage2, appId: str, networkingFile: str, envFile: str) -> ContainerStage2:
    # Strip leading/trailing whitespace from container.name
    container.name = container.name.strip()
    # If the name still contains a newline, throw an error
    if container.name.find("\n") != -1:
        raise Exception("Newline in container name")
    env_var = "APP_{}_{}_IP".format(
        appId.upper().replace("-", "_"),
        container.name.upper().replace("-", "_")
    )
    # Write a list of used IPs to the usedIpFile as JSON, and read that file to check if an IP
    # can be used
    usedIps = []
    networkingData = {}
    if path.isfile(networkingFile):
        with open(networkingFile, 'r') as f:
            networkingData = json.load(f)
    if 'ip_addresses' in networkingData:
        usedIps = list(networkingData['ip_addresses'].values())
    else:
        networkingData['ip_addresses'] = {}
    # An IP 10.21.21.xx, with x being a random number above 40 is asigned to the container
    # If the IP is already in use, it will be tried again until it's not in use
    # If it's not in use, it will be added to the usedIps list and written to the usedIpFile
    # If the usedIpsFile contains all IPs between  10.21.21.20 and  10.21.21.255 (inclusive),
    # Throw an error, because no more IPs can be used
    if len(usedIps) == 235:
        raise Exception("No more IPs can be used")
    if "{}-{}".format(appId, container.name) in networkingData['ip_addresses']:
        ip = networkingData['ip_addresses']["{}-{}".format(
            appId, container.name)]
    else:
        while True:
            ip = "10.21.21." + str(random.randint(20, 255))
            if ip not in usedIps:
                networkingData['ip_addresses']["{}-{}".format(
                    appId, container.name)] = ip
                break
    container.networks = from_dict(data_class=NetworkConfig, data={'default': {
        'ipv4_address': "$" + env_var}})
    dotEnv = parse_dotenv(envFile)
    if env_var in dotEnv and str(dotEnv[env_var]) == str(ip):
        return container
    # Now append a new line  with APP_{app_name}_{container_name}_IP=${IP} to the envFile
    with open(envFile, 'a') as f:
        f.write("{}={}\n".format(env_var, ip))
    with open(networkingFile, 'w') as f:
        json.dump(networkingData, f)
    return container
 def assignPort(container: dict, appId: str, networkingFile: str, envFile: str):
    # Strip leading/trailing whitespace from container.name
    container.name = container.name.strip()
    # If the name still contains a newline, throw an error
    if container.name.find("\n") != -1 or container.name.find(" ") != -1:
        raise Exception("Newline or space in container name")
    env_var = "APP_{}_{}_PORT".format(
        appId.upper().replace("-", "_"),
        container.name.upper().replace("-", "_")
    )
    port = getFreePort(networkingFile, appId)
    dotEnv = parse_dotenv(envFile)
    if env_var in dotEnv and str(dotEnv[env_var]) == str(port):
        return {"port": port, "env_var": "${{{}}}".format(env_var)}
    # Now append a new line  with APP_{app_name}_{container_name}_PORT=${PORT} to the envFile
    with open(envFile, 'a') as f:
        f.write("{}={}\n".format(env_var, port))
    # This is confusing, but {{}} is an escaped version of {} so it is ${{ {} }}
    # where the outer {{ }} will be replaced by {} in the returned string
    return {"port": port, "env_var": "${{{}}}".format(env_var)}
 def getMainContainer(app: dict):
    if len(app.containers) == 1:
        return app.containers[0]
    else:
        if not app.metadata.mainContainer:
            app.metadata.mainContainer = 'main'
        for container in app.containers:
            if container.name == app.metadata.mainContainer:
                return container
    raise Exception(
        "No main container found for app {}".format(app.metadata.name))
 def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
    registryFile = path.join(nodeRoot, "apps", "registry.json")
    registry: list = []
    if path.isfile(registryFile):
        with open(registryFile, 'r') as f:
            registry = json.load(f)
    else:
        raise Exception("Registry file not found")
    dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
    mainContainer = getMainContainer(app)
    portDetails = assignPort(mainContainer, app.metadata.id, path.join(
        nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
    containerPort = portDetails['port']
    portAsEnvVar = portDetails['env_var']
    portToAppend = portAsEnvVar
    mainPort = False
    if mainContainer.port:
        portToAppend = "{}:{}".format(portAsEnvVar, mainContainer.port)
        mainPort = mainContainer.port
        del mainContainer.port
    else:
        portToAppend = "{}:{}".format(portAsEnvVar, portAsEnvVar)
    if mainContainer.ports:
        mainContainer.ports.append(portToAppend)
        # Set the main port to the first port in the list, if it contains a :, it's the port after the :
        # If it doesn't contain a :, it's the port itself
        if mainPort == False:
            mainPort = mainContainer.ports[0]
            if mainPort.find(":") != -1:
                mainPort = mainPort.split(":")[1]
    else:
        mainContainer.ports = [portToAppend]
        if mainPort == False:
            mainPort = portDetails['port']
    mainContainer = assignIp(mainContainer, app.metadata.id, path.join(
        nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
    # If the IP wasn't in dotenv before, now it should be
    dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
    containerIP = dotEnv['APP_{}_{}_IP'.format(app.metadata.id.upper().replace(
        "-", "_"), mainContainer.name.upper().replace("-", "_"))]
    hiddenservice = getHiddenService(
        app.metadata.name, app.metadata.id, containerIP, mainPort)
    torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
    torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
    with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
        f.write(hiddenservice)
    # Also set the port in metadata
    app.metadata.port = int(containerPort)
    for registryApp in registry:
        if registryApp['id'] == app.metadata.id:
            registry[registry.index(registryApp)]['port'] = int(containerPort)
            break
    with open(registryFile, 'w') as f:
        json.dump(registry, f, indent=4, sort_keys=True)
    return app
 def configureIps(app: AppStage2, networkingFile: str, envFile: str):
    for container in app.containers:
        if container.noNetwork:
            # Check if port is defined for the container
            if container.port:
                raise Exception("Port defined for container without network")
            if app.metadata.mainContainer == container.name:
                raise Exception("Main container without network")
            # Skip this iteration of the loop
            continue
        container = assignIp(container, app.metadata.id,
                             networkingFile, envFile)
    return app
 def configureHiddenServices(app: dict, nodeRoot: str) -> None:
    dotEnv = parse_dotenv(path.join(nodeRoot, ".env"))
    hiddenServices = ""
    if len(app.containers) == 1:
        mainContainer = app.containers[0]
    else:
        mainContainer = None
        if app.metadata.mainContainer == None:
            app.metadata.mainContainer = 'main'
        for container in app.containers:
            if container.name == app.metadata.mainContainer:
                mainContainer = container
                break
        if mainContainer is None:
            raise Exception("No main container found")
    for container in app.containers:
        env_var = "APP_{}_{}_IP".format(
            app.metadata.id.upper().replace("-", "_"),
            container.name.upper().replace("-", "_")
        )
        hiddenServices += getContainerHiddenService(
            app.metadata.name, app.metadata.id, container, dotEnv[env_var], container.name == mainContainer.name)
    torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
    torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
    with open(path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
        f.write(hiddenServices)
--- a/app/lib/composegenerator/v1/types.py
+++ b/app/lib/composegenerator/v1/types.py
@ -1,151 +0,0 @@
 from typing import Union, List
 from dataclasses import dataclass, field
 from dacite import from_dict
@dataclass
 class Metadata:
    id: str
    name: str
    version: str
    category: str
    tagline: str
    description: str
    developer: str
    website: str
    repo: str
    support: str
    gallery: List[str] = field(default_factory=list)
    dependencies: List[str] = field(default_factory=list)
    mainContainer: Union[str, None] = None
    updateContainer: Union[str, None] = None
    path: str = ""
    defaultPassword: str = ""
    torOnly: bool = False
@dataclass
 class Container:
    name: str
    image: str
    permissions: list = field(default_factory=list)
    ports: list = field(default_factory=list)
    port: Union[int, None] = None
    environment: Union[dict, None] = None
    data: list = field(default_factory=list)
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
    depends_on: list = field(default_factory=list)
    entrypoint: Union[List[str], str] = field(default_factory=list)
    bitcoin_mount_dir: Union[str, None] = None
    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
    needsHiddenService: Union[bool, None] = None
    hiddenServicePort: Union[int, None] = None
    hiddenServicePorts: Union[dict, None] = None
    environment_allow: list = field(default_factory=list)
    # Only added later
    volumes: list = field(default_factory=list)
    restart: Union[str, None] = None
@dataclass
 class App:
    version: Union[str, int]
    metadata: Metadata
    containers: List[Container]
 # Generate an app instance from an app dict
 def generateApp(appDict):
    return from_dict(data_class=App, data=appDict)
@dataclass
 class Network:
    ipv4_address: Union[str, None] = None
@dataclass
 class NetworkConfig:
    default: Network
 # After converting data dir and defining volumes, stage 2
@dataclass
 class ContainerStage2:
    id: str
    name: str
    image: str
    permissions: List[str] = field(default_factory=list)
    ports: list = field(default_factory=list)
    environment: Union[dict, None] = None
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
    depends_on: List[str] = field(default_factory=list)
    entrypoint: Union[List[str], str] = field(default_factory=list)
    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
    needsHiddenService: Union[bool, None] = None
    hiddenServicePort: Union[int, None] = None
    hiddenServicePorts: Union[dict, None] = None
    volumes: List[str] = field(default_factory=list)
    networks: NetworkConfig = field(default_factory=NetworkConfig)
    restart: Union[str, None] = None
@dataclass
 class AppStage2:
    version: Union[str, int]
    metadata: Metadata
    containers: List[ContainerStage2]
@dataclass
 class MetadataStage3:
    id: str
    name: str
    version: str
    category: str
    tagline: str
    description: str
    developer: str
    website: str
    dependencies: List[str]
    repo: str
    support: str
    gallery: List[str]
    mainContainer: Union[str, None] = None
    updateContainer: Union[str, None] = None
    path: str = ""
    defaultPassword: str = ""
    torOnly: bool = False
@dataclass
 class AppStage3:
    version: Union[str, int]
    metadata: MetadataStage3
    containers: List[ContainerStage2]
@dataclass
 class ContainerStage4:
    id: str
    name: str
    image: str
    ports: list = field(default_factory=list)
    environment: Union[dict, None] = None
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
    depends_on: List[str] = field(default_factory=list)
    entrypoint: Union[List[str], str] = field(default_factory=list)
    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
    needsHiddenService: Union[bool, None] = None
    hiddenServicePort: Union[int, None] = None
    hiddenServicePorts: Union[dict, None] = None
    volumes: List[str] = field(default_factory=list)
    networks: NetworkConfig = field(default_factory=NetworkConfig)
    restart: Union[str, None] = None
@dataclass
 class AppStage4:
    version: Union[str, int]
    metadata: MetadataStage3
    services: List[ContainerStage4]
--- a/app/lib/composegenerator/v1/utils/networking.py
+++ b/app/lib/composegenerator/v1/utils/networking.py
@ -1,118 +0,0 @@
 # SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 import json
 import os
 import random
 from lib.composegenerator.v1.types import Container
 def getFreePort(networkingFile: str, appId: str):
    # Ports used currently in Citadel
    usedPorts = [
        # Dashboard
        80,
        # Sometimes used by nginx with some setups
        433,
        # Dashboard SSL
        443,
        # Bitcoin Core P2P
        8333,
        # LND gRPC
        10009,
        # LND REST
        8080,
        # Electrum Server
        50001,
        # Tor Proxy
        9050,
    ]
    networkingData = {}
    if os.path.isfile(networkingFile):
        with open(networkingFile, 'r') as f:
            networkingData = json.load(f)
    if 'ports' in networkingData:
        usedPorts += list(networkingData['ports'].values())
    else:
        networkingData['ports'] = {}
    if appId in networkingData['ports']:
        return networkingData['ports'][appId]
    while True:
        port = str(random.randint(1024, 49151))
        if port not in usedPorts:
            # Check if anyhing is listening on the specific port
            if os.system("netstat -ntlp | grep " + port + " > /dev/null") != 0:
                networkingData['ports'][appId] = port
                break
    with open(networkingFile, 'w') as f:
        json.dump(networkingData, f)
    return port
 def getHiddenServiceMultiPort(name: str, id: str, internalIp: str, ports: list) -> str:
    hiddenServices = '''
 # {} Hidden Service
 HiddenServiceDir /var/lib/tor/app-{}
 '''.format(name, id)
    for port in ports:
        hiddenServices += 'HiddenServicePort {} {}:{}'.format(
            port, internalIp, port)
        hiddenServices += "\n"
    return hiddenServices
 def getHiddenServiceString(name: str, id: str, internalPort, internalIp: str, publicPort) -> str:
    return '''
 # {} Hidden Service
 HiddenServiceDir /var/lib/tor/app-{}
 HiddenServicePort {} {}:{}
 '''.format(name, id, publicPort, internalIp, internalPort)
 def getHiddenService(appName: str, appId: str, appIp: str, appPort: str) -> str:
    return getHiddenServiceString(appName, appId, appPort, appIp, "80")
 def getContainerHiddenService(appName: str, appId: str, container: Container, containerIp: str, isMainContainer: bool) -> str:
    if not container.needsHiddenService and not isMainContainer:
        return ""
    if (container.ports or not container.port) and not container.hiddenServicePort and not isMainContainer:
        print("Container {} for app {} isn't compatible with hidden service assignment".format(
            container.name, appName))
        return ""
    if isMainContainer:
        if not container.hiddenServicePorts:
            return ""
        # hiddenServicePorts is a map of hidden service name to port
        # We need to generate a hidden service for each one
        hiddenServices = ""
        for name, port in container.hiddenServicePorts.items():
            if ".." in name:
                print(".. Not allowed in service names, this app ({}) isn't getting a hidden service.".format(appName))
            # If port is a list, use getHiddenServiceMultiPort
            if isinstance(port, list):
                hiddenServices += getHiddenServiceMultiPort("{} {}".format(appName, name), "{}-{}".format(
                    appId, name), containerIp, port)
            else:
                hiddenServices += getHiddenServiceString("{} {}".format(appName, name), "{}-{}".format(
                    appId, name), port, containerIp, port)
        del container.hiddenServicePorts
        return hiddenServices
    del container.needsHiddenService
    if not container.port:
        data = getHiddenServiceString(appName + container.name, "{}-{}".format(
            appId, container.name), container.hiddenServicePort, containerIp, "80")
        del container.hiddenServicePort
        return data
    else:
        return getHiddenServiceString(appName + container.name, "{}-{}".format(
            appId, container.name), container.port, containerIp, container.port)
--- a/app/lib/composegenerator/v2/networking.py
+++ b/app/lib/composegenerator/v2/networking.py
@ -8,7 +8,7 @@ import json
 from os import path
 import random
 from lib.composegenerator.v2.utils.networking import getContainerHiddenService
-from lib.composegenerator.v1.networking import assignIp, assignPort
+from lib.composegenerator.shared.networking import assignIp, assignPort
 def getMainContainer(app: App) -> Container:
--- a/app/lib/composegenerator/v3/networking.py
+++ b/app/lib/composegenerator/v3/networking.py
@ -7,8 +7,7 @@ from lib.citadelutils import parse_dotenv
 import json
 from os import path
 import random
-from lib.composegenerator.v1.networking import assignIp, assignPort
+from lib.composegenerator.shared.networking import assignIp, assignPort
 def getMainContainerIndex(app: App):
    if len(app.containers) == 1:
@ -105,4 +104,3 @@ def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
    with open(envFile, 'a') as f:
        f.write("{}={}\n".format(portAsEnvVar, app.metadata.port))
    return app
--- a/app/lib/manage.py
+++ b/app/lib/manage.py
@ -6,9 +6,11 @@ import stat
 import sys
 import tempfile
 import threading
 import random
 from typing import List
 from sys import argv
 import os
 import fcntl
 import requests
 import shutil
 import json
@ -25,16 +27,37 @@ except Exception:
    print("Continuing anyway, but some features won't be available,")
    print("for example checking for app updates")
 from lib.composegenerator.v1.generate import createComposeConfigFromV1
 from lib.composegenerator.v2.generate import createComposeConfigFromV2
 from lib.composegenerator.v3.generate import createComposeConfigFromV3
 from lib.validate import findAndValidateApps
 from lib.metadata import getAppRegistry
 from lib.entropy import deriveEntropy
 class FileLock:
    """Implements a file-based lock using flock(2).
    The lock file is saved in directory dir with name lock_name.
    dir is the current directory by default.
    """
    def __init__(self, lock_name, dir="."):
        self.lock_file = open(os.path.join(dir, lock_name), "w")
    def acquire(self, blocking=True):
        """Acquire the lock.
        If the lock is not already acquired, return None.  If the lock is
        acquired and blocking is True, block until the lock is released.  If
        the lock is acquired and blocking is False, raise an IOError.
        """
        ops = fcntl.LOCK_EX
        if not blocking:
            ops |= fcntl.LOCK_NB
        fcntl.flock(self.lock_file, ops)
    def release(self):
        """Release the lock. Return None even if lock not currently acquired"""
        fcntl.flock(self.lock_file, fcntl.LOCK_UN)
 # For an array of threads, join them and wait for them to finish
 def joinThreads(threads: List[threading.Thread]):
    for thread in threads:
        thread.join()
@ -50,26 +73,58 @@ updateIgnore = os.path.join(appsDir, ".updateignore")
 appDataDir = os.path.join(nodeRoot, "app-data")
 userFile = os.path.join(nodeRoot, "db", "user.json")
 legacyScript = os.path.join(nodeRoot, "scripts", "app")
 with open(os.path.join(nodeRoot, "db", "dependencies.yml"), "r") as file: 
  dependencies = yaml.safe_load(file)
 # Returns a list of every argument after the second one in sys.argv joined into a string by spaces
 def getArguments():
    arguments = ""
    for i in range(3, len(argv)):
        arguments += argv[i] + " "
    return arguments
 def handleAppV4(app):
    composeFile = os.path.join(appsDir, app, "docker-compose.yml")
    os.chown(os.path.join(appsDir, app), 1000, 1000)
    os.system("docker run --rm -v {}:/apps -u 1000:1000 {} /app-cli convert --app-name '{}' --port-map /apps/ports.json /apps/{}/app.yml /apps/{}/result.yml --services 'lnd'".format(appsDir, dependencies['app-cli'], app, app, app))
    with open(os.path.join(appsDir, app, "result.yml"), "r") as resultFile:
        resultYml = yaml.safe_load(resultFile)
    with open(composeFile, "w") as dockerComposeFile:
        yaml.dump(resultYml["spec"], dockerComposeFile)
    torDaemons = ["torrc-apps", "torrc-apps-2", "torrc-apps-3"]
    torFileToAppend = torDaemons[random.randint(0, len(torDaemons) - 1)]
    with open(os.path.join(nodeRoot, "tor", torFileToAppend), 'a') as f:
        f.write(resultYml["new_tor_entries"])
    mainPort = resultYml["port"]
    registryFile = os.path.join(nodeRoot, "apps", "registry.json")
    registry: list = []
    lock = FileLock("citadeL_registry_lock", dir="/tmp")
    lock.acquire()
    if os.path.isfile(registryFile):
        with open(registryFile, 'r') as f:
            registry = json.load(f)
    else:
        raise Exception("Registry file not found")
    for registryApp in registry:
        if registryApp['id'] == app:
            registry[registry.index(registryApp)]['port'] = resultYml["port"]
            break
    with open(registryFile, 'w') as f:
        json.dump(registry, f, indent=4, sort_keys=True)
    lock.release()
 def getAppYml(name):
    with open(os.path.join(appsDir, "sourceMap.json"), "r") as f:
        sourceMap = json.load(f)
    if not name in sourceMap:
-        print("Warning: App {} is not in the source map".format(name))
+        print("Warning: App {} is not in the source map".format(name), file=sys.stderr)
        sourceMap = {
            name: {
-                "githubRepo": "runcitadel/core",
+                "githubRepo": "runcitadel/apps",
-                "branch": "v2"
+                "branch": "v4-stable"
            }
        }
    url = 'https://raw.githubusercontent.com/{}/{}/apps/{}/app.yml'.format(sourceMap[name]["githubRepo"], sourceMap[name]["branch"], name)
@ -89,16 +144,31 @@ def update(verbose: bool = False):
        json.dump(registry["ports"], f, sort_keys=True)
    print("Wrote registry to registry.json")
    os.system("docker pull {}".format(dependencies['app-cli']))
    threads = list()
    # Loop through the apps and generate valid compose files from them, then put these into the app dir
    for app in apps:
-        composeFile = os.path.join(appsDir, app, "docker-compose.yml")
+        try:
-        appYml = os.path.join(appsDir, app, "app.yml")
+            composeFile = os.path.join(appsDir, app, "docker-compose.yml")
-        with open(composeFile, "w") as f:
+            appYml = os.path.join(appsDir, app, "app.yml")
-            appCompose = getApp(appYml, app)
+            with open(appYml, 'r') as f:
-            if appCompose:
+                appDefinition = yaml.safe_load(f)
-                f.write(yaml.dump(appCompose, sort_keys=False))
+            if 'citadel_version' in appDefinition:
-                if verbose:
+                thread = threading.Thread(target=handleAppV4, args=(app,))
-                    print("Wrote " + app + " to " + composeFile)
+                thread.start()
                threads.append(thread)
            else:
                appCompose = getApp(appDefinition, app)
                with open(composeFile, "w") as f:
                    if appCompose:
                        f.write(yaml.dump(appCompose, sort_keys=False))
                        if verbose:
                            print("Wrote " + app + " to " + composeFile)
        except Exception as err:
            print("Failed to convert app {}".format(app))
            print(err)
    joinThreads(threads)
    print("Generated configuration successfully")
@ -118,22 +188,29 @@ def getUserData():
            userData = json.load(f)
    return userData
-def checkUpdateAvailable(name: str) -> bool:
+def checkUpdateAvailable(name: str):
    latestAppYml = yaml.safe_load(getAppYml(name))
    with open(os.path.join(appsDir, name, "app.yml"), "r") as f:
        originalAppYml = yaml.safe_load(f)
    if not "metadata" in latestAppYml or not "version" in latestAppYml["metadata"] or not "metadata" in originalAppYml or not "version" in originalAppYml["metadata"]:
-        print("App {} is not valid".format(name))
+        print("App {} is not valid".format(name), file=sys.stderr)
        return False
    if semver.compare(latestAppYml["metadata"]["version"], originalAppYml["metadata"]["version"]) > 0:
        return {
            "updateFrom": originalAppYml["metadata"]["version"],
            "updateTo": latestAppYml["metadata"]["version"]
        }
    else:
        return False
    return semver.compare(latestAppYml["metadata"]["version"], originalAppYml["metadata"]["version"]) > 0
 def getAvailableUpdates():
-    availableUpdates = []
+    availableUpdates = {}
    apps = findAndValidateApps(appsDir)
    for app in apps:
        try:
-            if checkUpdateAvailable(app):
+            checkResult = checkUpdateAvailable(app)
-                availableUpdates.append(app)
+            if checkResult:
                availableUpdates[app] = checkResult
        except Exception:
            print("Warning: Can't check app {} yet".format(app), file=sys.stderr)
    return availableUpdates
@ -178,23 +255,16 @@ def stopInstalled():
    joinThreads(threads)
 # Loads an app.yml and converts it to a docker-compose.yml
-
+def getApp(app, appId: str):
 def getApp(appFile: str, appId: str):
    with open(appFile, 'r') as f:
        app = yaml.safe_load(f)
    if not "metadata" in app:
        raise Exception("Error: Could not find metadata in " + appFile)
    app["metadata"]["id"] = appId
-    if 'version' in app and str(app['version']) == "1":
+    if 'version' in app and str(app['version']) == "2":
        print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId))
        return createComposeConfigFromV1(app, nodeRoot)
    elif 'version' in app and str(app['version']) == "2":
        print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId))
        return createComposeConfigFromV2(app, nodeRoot)
    elif 'version' in app and str(app['version']) == "3":
        print("Warning: App {} uses version 3 of the app.yml format, which is scheduled for removal in Citadel 0.3.0".format(appId))
        return createComposeConfigFromV3(app, nodeRoot)
    else:
        raise Exception("Error: Unsupported version of app.yml")
--- a/app/lib/metadata.py
+++ b/app/lib/metadata.py
@ -4,10 +4,10 @@
 import os
 import yaml
 import traceback
 from lib.composegenerator.next.stage1 import createCleanConfigFromV3
 from lib.composegenerator.v2.networking import getMainContainer
-from lib.composegenerator.v1.networking import getFreePort
+from lib.composegenerator.shared.networking import assignIpV4
 from lib.entropy import deriveEntropy
 from typing import List
 import json
@ -41,11 +41,15 @@ def getAppRegistry(apps, app_path):
    app_metadata = []
    for app in apps:
        app_yml_path = os.path.join(app_path, app, 'app.yml')
        app_cache_path = os.path.join(app_path, app, 'app.cache.json')
        if os.path.isfile(app_yml_path):
            try:
                with open(app_yml_path, 'r') as f:
                    app_yml = yaml.safe_load(f.read())
                version = False
                if 'version' in app_yml:
                    version = int(app_yml['version'])
                elif 'citadel_version' in app_yml:
                    version = int(app_yml['citadel_version'])
                metadata: dict = app_yml['metadata']
                metadata['id'] = app
                metadata['path'] = metadata.get('path', '')
@ -55,14 +59,14 @@ def getAppRegistry(apps, app_path):
                if "mainContainer" in metadata:
                    metadata.pop("mainContainer")
                app_metadata.append(metadata)
-                if(app_yml["version"] != 3):
+                if version < 3:
                    getPortsOldApp(app_yml, app)
-                else:
+                elif version == 3:
                    getPortsV3App(app_yml, app)
-                    with open(app_cache_path, 'w') as f:
+                elif version == 4:
-                        json.dump(createCleanConfigFromV3(app_yml, os.path.dirname(app_path)), f)
+                    getPortsV4App(app_yml, app)
            except Exception as e:
-                print(e)
+                print(traceback.format_exc())
                print("App {} is invalid!".format(app))
    appPortsToMap()
    return {
@ -97,12 +101,12 @@ def getNewPort(usedPorts):
        lastPort2 = lastPort2 + 1
    return lastPort2
-def validatePort(appContainer, port, appId, priority: int, isDynamic = False): 
+def validatePort(containerName, appContainer, port, appId, priority: int, isDynamic = False): 
    if port not in appPorts and port not in citadelPorts and port != 0:
        appPorts[port] = {
            "app": appId,
            "port": port,
-            "container": appContainer["name"],
+            "container": containerName,
            "priority": priority,
            "dynamic": isDynamic,
        }
@ -115,7 +119,7 @@ def validatePort(appContainer, port, appId, priority: int, isDynamic = False):
                appPorts[port]  = {
                    "app": appId,
                    "port": port,
-                    "container": appContainer["name"],
+                    "container": containerName,
                    "priority": priority,
                    "dynamic": isDynamic,
                }
@ -128,7 +132,7 @@ def validatePort(appContainer, port, appId, priority: int, isDynamic = False):
                    appPorts[newPort]  = {
                        "app": appId,
                        "port": port,
-                        "container": appContainer["name"],
+                        "container": containerName,
                        "priority": priority,
                        "dynamic": isDynamic,
                    }
@ -136,28 +140,44 @@ def validatePort(appContainer, port, appId, priority: int, isDynamic = False):
 def getPortsOldApp(app, appId):
    for appContainer in app["containers"]:
        if "port" in appContainer:
-            validatePort(appContainer, appContainer["port"], appId, 0)
+            validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0)
        if "ports" in appContainer:
            for port in appContainer["ports"]:
                realPort = int(str(port).split(":")[0])
-                validatePort(appContainer, realPort, appId, 2)
+                validatePort(appContainer["name"], appContainer, realPort, appId, 2)
 def getPortsV3App(app, appId):
    for appContainer in app["containers"]:
        if "port" in appContainer:
            if "preferredOutsidePort" in appContainer and "requiresPort" in appContainer and appContainer["requiresPort"]:
-                validatePort(appContainer, appContainer["preferredOutsidePort"], appId, 2)
+                validatePort(appContainer["name"], appContainer, appContainer["preferredOutsidePort"], appId, 2)
            elif "preferredOutsidePort" in appContainer:
-                validatePort(appContainer, appContainer["preferredOutsidePort"], appId, 1)
+                validatePort(appContainer["name"], appContainer, appContainer["preferredOutsidePort"], appId, 1)
            else:
-                validatePort(appContainer, appContainer["port"], appId, 0)
+                validatePort(appContainer["name"], appContainer, appContainer["port"], appId, 0)
        elif "requiredPorts" not in appContainer and "requiredUdpPorts" not in appContainer:
-                validatePort(appContainer, getNewPort(appPorts.keys()), appId, 0, True)
+                validatePort(appContainer["name"], appContainer, getNewPort(appPorts.keys()), appId, 0, True)
        if "requiredPorts" in appContainer:
            for port in appContainer["requiredPorts"]:
-                validatePort(appContainer, port, appId, 2)
+                validatePort(appContainer["name"], appContainer, port, appId, 2)
        if "requiredUdpPorts" in appContainer:
            for port in appContainer["requiredUdpPorts"]:
-                validatePort(appContainer, port, appId, 2)
+                validatePort(appContainer["name"], appContainer, port, appId, 2)
 def getPortsV4App(app, appId):
    for appContainerName in app["services"].keys():
        appContainer = app["services"][appContainerName]
        if "enable_networking" in appContainer and not appContainer["enable_networking"]:
            return
        assignIpV4(appId, appContainerName)
        if "port" in appContainer:
            validatePort(appContainerName, appContainer, appContainer["port"], appId, 0)
        if "required_ports" in appContainer:
            if "tcp" in appContainer["required_ports"]:
                for port in appContainer["required_ports"]["tcp"].keys():
                    validatePort(appContainerName, appContainer, port, appId, 2)
            if "udp" in appContainer["required_ports"]:
                for port in appContainer["required_ports"]["udp"].keys():
                    validatePort(appContainerName, appContainer, port, appId, 2)
--- a/app/lib/validate.py
+++ b/app/lib/validate.py
@ -6,11 +6,10 @@ import os
 import yaml
 from jsonschema import validate
 import yaml
 import traceback
 scriptDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
 with open(os.path.join(scriptDir, 'app-standard-v1.yml'), 'r') as f:
    schemaVersion1 = yaml.safe_load(f)
 with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f:
    schemaVersion2 = yaml.safe_load(f)
 with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f:
@ -19,21 +18,13 @@ with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f:
 # Validates app data
 # Returns true if valid, false otherwise
 def validateApp(app: dict):
-    if 'version' in app and str(app['version']) == "1":
+    if 'version' in app and str(app['version']) == "2":
        try:
            validate(app, schemaVersion1)
            return True
        # Catch and log any errors, and return false
        except Exception as e:
            print(e)
            return False
    elif 'version' in app and str(app['version']) == "2":
        try:
            validate(app, schemaVersion2)
            return True
        # Catch and log any errors, and return false
        except Exception as e:
-            print(e)
+            print(traceback.format_exc())
            return False
    elif 'version' in app and str(app['version']) == "3":
        try:
@ -41,12 +32,13 @@ def validateApp(app: dict):
            return True
        # Catch and log any errors, and return false
        except Exception as e:
-            print(e)
+            print(traceback.format_exc())
            return False
-    else:
+    elif 'version' not in app and 'citadel_version' not in app:
        print("Unsupported app version")
        return False
-
+    else:
        return True
 # Read in an app.yml file and pass it to the validation function
 # Returns true if valid, false otherwise
@ -72,14 +64,17 @@ def findApps(dir: str):
 def findAndValidateApps(dir: str):
    apps = []
    app_data = {}
-    for root, dirs, files in os.walk(dir, topdown=False):
+    for subdir in os.scandir(dir):
-        for name in dirs:
+        if not subdir.is_dir():
-            app_dir = os.path.join(root, name)
+            continue
-            if os.path.isfile(os.path.join(app_dir, "app.yml")):
+        app_dir = subdir.path
-                apps.append(name)
+        if os.path.isfile(os.path.join(app_dir, "app.yml")):
-                # Read the app.yml and append it to app_data
+            apps.append(subdir.name)
-                with open(os.path.join(app_dir, "app.yml"), 'r') as f:
+            # Read the app.yml and append it to app_data
-                    app_data[name] = yaml.safe_load(f)
+            with open(os.path.join(app_dir, "app.yml"), 'r') as f:
                app_data[subdir.name] = yaml.safe_load(f)
        else:
            print("App {} has no app.yml".format(subdir.name))
    # Now validate all the apps using the validateAppFile function by passing the app.yml as an argument to it, if an app is invalid, remove it from the list
    for app in apps:
        appyml = app_data[app]
@ -113,12 +108,13 @@ def findAndValidateApps(dir: str):
                            should_continue=False
        if not should_continue:
            continue
-        for container in appyml['containers']:
+        if 'containers' in appyml:
-            if 'permissions' in container:
+            for container in appyml['containers']:
-                for permission in container['permissions']:
+                if 'permissions' in container:
-                    if permission not in appyml['metadata']['dependencies'] and permission not in ["root", "hw"]:
+                    for permission in container['permissions']:
-                        print("WARNING: App {}'s container '{}' requires the '{}' permission, but the app doesn't list it in it's dependencies".format(app, container['name'], permission))
+                        if permission not in appyml['metadata']['dependencies'] and permission not in ["root", "hw"]:
-                        apps.remove(app)
+                            print("WARNING: App {}'s container '{}' requires the '{}' permission, but the app doesn't list it in it's dependencies".format(app, container['name'], permission))
-                        # Skip to the next iteration of the loop
+                            apps.remove(app)
-                        continue
+                            # Skip to the next iteration of the loop
                            continue
    return apps
--- a/bin/citadel
+++ b/bin/citadel
@ -0,0 +1 @@
 ../cli/citadel
--- a/cli/citadel
+++ b/cli/citadel
@ -0,0 +1,487 @@
 #!/usr/bin/env bash
 # SPDX-FileCopyrightText: 2022 Citadel and contributors
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 set -euo pipefail
 CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)"
 CLI_NAME="$(basename $0)"
 CLI_VERSION="0.0.1"
 CLI_DIR="$(dirname "$(readlink -f "$0")")"
 SERVICE_NAME="citadel-startup"
 EDITOR="${EDITOR:-micro}"
 source $CLI_DIR/utils/functions.sh
 source $CLI_DIR/utils/multiselect.sh
 source $CLI_DIR/utils/spinner.sh
 source $CLI_DIR/utils/helpers.sh
 if [ -z ${1+x} ]; then
  command=""
 else
  command="$1"
 fi
 # Check Citadel Status
 if [[ "$command" = "status" ]]; then
  POSITIONAL_ARGS=()
  long=false
  while [[ $# -gt 0 ]]; do
    case $1 in
    -l | --long)
      long=true
      shift
      ;;
    -* | --*)
      echo "Unknown option $1"
      exit 1
      ;;
    *)
      POSITIONAL_ARGS+=("$1")
      shift
      ;;
    esac
  done
  set -- "${POSITIONAL_ARGS[@]}"
  free -m | awk 'NR==2{printf "Memory Usage: %s/%sMB (%.2f%%)\n", $3,$2,$3*100/$2 }'
  df -h | awk '$NF=="/"{printf "Disk Usage: %d/%dGB (%s)\n", $3,$2,$5}'
  top -bn1 | grep load | awk '{printf "CPU Load: %.2f\n", $(NF-2)}'
  echo
  if $long; then
    docker container ls --format "table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"
  else
    docker container ls --format "table {{.Names}}\t{{.Status}}"
  fi
  if [[ $(pgrep -f karen) ]]; then
    printf "\nKaren is listening.\n"
  else
    printf "\nERROR: Karen is not listening.\n"
  fi
  exit
 fi
 # Update Citadel
 if [[ "$command" = "update" ]]; then
  POSITIONAL_ARGS=()
  branch=$(get_update_channel)
  force=false
  while [[ $# -gt 0 ]]; do
    case $1 in
    -b | --branch)
      branch="$2"
      shift # past argument
      shift # past value
      ;;
    --force)
      force=true
      shift # past argument
      ;;
    -* | --*)
      echo "Unknown option $1"
      exit 1
      ;;
    *)
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift                   # past argument
      ;;
    esac
  done
  set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
  if $force; then
    sudo rm -f $CITADEL_ROOT/statuses/update-in-progress
  fi
  sudo $CITADEL_ROOT/scripts/update/update --repo runcitadel/core#$branch
  exit
 fi
 # Start Citadel
 if [[ "$command" = "start" ]]; then
  if $(is_managed_by_systemd); then
    if $(is_service_active); then
      echo 'Citadel is already running.'
    else
      sudo systemctl start citadel-startup
    fi
  else
    sudo $CITADEL_ROOT/scripts/start
  fi
  exit
 fi
 # Stop Citadel
 if [[ "$command" = "stop" ]]; then
  active=$(is_service_active)
  if $(is_managed_by_systemd) && $active; then
    if $(is_service_active); then
      sudo systemctl stop citadel-startup
    else
      echo 'Citadel is not running.'
    fi
  else
    sudo $CITADEL_ROOT/scripts/stop
  fi
  exit
 fi
 # Restart Citadel
 if [[ "$command" = "restart" ]]; then
  shift
  # TODO: enable restarting services
  if [ ! -z ${1+x} ]; then
    echo "Too many arguments."
    echo "Usage: \`$CLI_NAME $command\`"
    exit 1
  fi
  if $(is_managed_by_systemd); then
    sudo systemctl restart $SERVICE_NAME
  else
    sudo $CITADEL_ROOT/scripts/stop
    sudo $CITADEL_ROOT/scripts/start
  fi
  exit
 fi
 # Reboot the system
 if [[ "$command" = "reboot" ]]; then
  $CLI_NAME stop || true
  sudo reboot
  exit
 fi
 # Shutdown the system
 if [[ "$command" = "shutdown" ]]; then
  $CLI_NAME stop || true
  sudo shutdown now
  exit
 fi
 # List all installed services apps
 if [[ "$command" = "list" ]]; then
  echo 'karen'
  docker ps --format "{{.Names}}"
  exit
 fi
 # Run a command inside a container
 if [[ "$command" = "run" ]]; then
  shift
  if [ -z ${1+x} ]; then
    echo "Specify an app or service."
    echo "Usage: \`$CLI_NAME $command <service> \"<command>\"\`"
    exit 1
  fi
  if [ -z ${2+x} ]; then
    echo "Specify a command to run."
    echo "Usage: \`$CLI_NAME $command <service> \"<command>\"\`"
    exit 1
  fi
  docker exec -t $1 sh -c "$2" || {
    echo "To see all installed services & apps use \`$CLI_NAME list\`"
    echo "Usage: \`$CLI_NAME $command <service> \"<command>\"\`"
    exit 1
  }
  exit
 fi
 # Configure Citadel
 if [[ "$command" = "set" ]]; then
  shift
  if [ -z ${1+x} ]; then
    echo "Missing subcommand."
    echo "Usage: \`$CLI_NAME $command <subcommand>\`"
    exit 1
  else
    subcommand="$1"
  fi
  # Switch update channel
  if [[ "$subcommand" = "update-channel" ]]; then
    if [ -z ${2+x} ]; then
      echo "Specify an update channel to switch to."
      echo "Usage: \`$CLI_NAME $subcommand <stable|beta|c-lightning>\`"
      exit 1
    fi
    case $2 in "stable" | "beta" | "c-lightning")
      # continue
      ;;
    *)
      echo "Not a valid update channel: \"$2\""
      exit 1
      ;;
    esac
    sudo $CITADEL_ROOT/scripts/set-update-channel $2
    $CLI_NAME update
    exit
  fi
  # Switch Bitcoin/Electrum implementation
  if [[ "$subcommand" = "implementation" ]] || [[ "$subcommand" = "impl" ]]; then
    shift
    sudo $CITADEL_ROOT/services/manage.py set $@
    $CLI_NAME restart
    exit
  fi
  # Switch Bitcoin network
  if [[ "$subcommand" = "network" ]]; then
    shift
    if [ -z ${1+x} ]; then
      echo "Specify a network to switch to."
      echo "Usage: \`$CLI_NAME $subcommand <mainnet|signet|testnet|regtest>\`"
    else
      case $1 in
      "mainnet" | "testnet" | "signet" | "regtest")
        sudo $CITADEL_ROOT/scripts/stop
        sudo OVERWRITE_NETWORK=$1 $CITADEL_ROOT/scripts/configure
        sudo $CITADEL_ROOT/scripts/start
        ;;
      *)
        echo "Not a valid value for network"
        exit 1
        ;;
      esac
    fi
    exit
  fi
  echo "\"$subcommand\" is not a valid subcommand."
  exit 1
 fi
 # App commands
 if [[ "$command" = "app" ]]; then
  shift
  sudo $CITADEL_ROOT/scripts/app $@
  exit
 fi
 # Edit common app configuration files
 if [[ "$command" = "configure" ]]; then
  if [ -z ${2+x} ]; then
    echo "Specify an app or service to configure."
    echo "Usage: \`$CLI_NAME $command <service>\`"
    exit 1
  fi
  POSITIONAL_ARGS=()
  persist=false
  while [[ $# -gt 0 ]]; do
    case $1 in
    --persist)
      persist=true
      shift # past argument
      ;;
    -* | --*)
      echo "Unknown option $1"
      exit 1
      ;;
    *)
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift                   # past argument
      ;;
    esac
  done
  set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
  # These service and app configs are already persisted
  # TODO: add more apps
  if [[ "$2" = "nextcloud" ]]; then
    edit_file --priviledged $CITADEL_ROOT/app-data/nextcloud/data/nextcloud/config/config.php
    prompt_apply_config nextcloud-web-1 false
    exit
  fi
  if [[ "$2" = "nginx" ]]; then
    edit_file $CITADEL_ROOT/nginx/nginx.conf
    prompt_apply_config nginx false
    exit
  fi
  if $persist; then
    echo "NOTE: As of now persisted config changes will not be kept when updating Citadel."
  else
    echo "NOTE: Some changes to this configuration file may be overwritten the next time you start Citadel."
    echo "To persist the changes run the command again with \`$CLI_NAME configure $2 --persist\`"
  fi
  read -p "Continue? [Y/n] " should_continue
  echo
  if [[ $should_continue =~ [Nn]$ ]]; then
    exit
  fi
  # Service and app configs below may be overwritten
  # TODO: check which implementation is running
  # and do "bitcoin" / "lightning" / "electrum"
  if [[ "$2" = "bitcoin" ]]; then
    if $persist; then
      edit_file $CITADEL_ROOT/templates/bitcoin-sample.conf
      prompt_apply_config bitcoin true
    else
      edit_file $CITADEL_ROOT/bitcoin/bitcoin.conf
      prompt_apply_config bitcoin false
    fi
    exit
  fi
  if [[ "$2" = "lnd" ]]; then
    if $persist; then
      edit_file $CITADEL_ROOT/templates/lnd-sample.conf
      prompt_apply_config lightning true
    else
      edit_file $CITADEL_ROOT/lnd/lnd.conf
      prompt_apply_config lightning false
    fi
    exit
  fi
  if [[ "$2" = "electrs" ]]; then
    edit_file $CITADEL_ROOT/templates/electrs-sample.toml
    prompt_apply_config electrum true
    exit
  fi
  if [[ "$2" = "fulcrumx" ]]; then
    if $persist; then
      edit_file $CITADEL_ROOT/templates/fulcrumx-sample.conf
      prompt_apply_config electrum true
    else
      edit_file $CITADEL_ROOT/fulcrumx/fulcrumx.conf
      prompt_apply_config electrum false
    fi
    exit
  fi
  echo "No service or app \"$2\" not found."
  exit 1
 fi
 # Show logs for apps & services
 if [[ "$command" = "logs" ]]; then
  shift
  POSITIONAL_ARGS=()
  follow=false
  while [[ $# -gt 0 ]]; do
    case $1 in
    -f | --follow)
      follow=true
      shift # past argument
      ;;
    -n | --tail)
      number_of_lines="$2"
      shift # past argument
      shift # past value
      ;;
    -* | --*)
      echo "Unknown option $1"
      exit 1
      ;;
    *)
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift                   # past argument
      ;;
    esac
  done
  set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
  # Set default number_of_lines if not set by user
  if [ -z ${number_of_lines+x} ]; then
    if [[ ${#POSITIONAL_ARGS[@]} == 0 ]] || [[ ${#POSITIONAL_ARGS[@]} == 1 ]]; then
      number_of_lines=40
    else
      number_of_lines=10
    fi
  fi
  if [ -z ${1+x} ] || [[ "$1" = "karen" ]]; then
    if [[ ${#POSITIONAL_ARGS[@]} == 2 ]]; then
      echo "Karen logs cannot be viewed together with other services."
      echo "Usage: \`$CLI_NAME $command karen\`"
      exit 1
    fi
    tail $($follow && echo "-f") -n $number_of_lines $CITADEL_ROOT/logs/karen.log
    exit
  fi
  if [[ ${#POSITIONAL_ARGS[@]} == 1 ]]; then
    docker logs $($follow && echo "-f") --tail $number_of_lines $@ || {
      echo "To see all installed services & apps use \`$CLI_NAME list\`"
      echo "Usage: \`$CLI_NAME $command <service>\`"
      exit 1
    }
  else
    # TODO: can only show logs for services in docker-compose.yml
    docker compose logs $($follow && echo "-f") --tail $number_of_lines $@ || {
      echo "To see all installed services & apps use \`$CLI_NAME list\`"
      echo "Usage: \`$CLI_NAME $command <service>\`"
      exit 1
    }
  fi
  exit
 fi
 # Debug Citadel
 if [[ "$command" = "debug" ]]; then
  shift
  sudo $CITADEL_ROOT/scripts/debug $@
  exit
 fi
 # Show version information for this CLI
 if [[ "$command" = "--version" ]] || [[ "$command" = "-v" ]]; then
  citadel_version=$(jq -r '.version' $CITADEL_ROOT/info.json)
  echo "Citadel v$citadel_version"
  echo "citadel-cli v$CLI_VERSION"
  exit
 fi
 # Show usage information for this CLI
 if [[ "$command" = "--help" ]] || [[ "$command" = "-h" ]]; then
  show_help
  exit
 fi
 # If we get here it means no valid command was supplied
 # Show help and exit
 show_help
 exit
--- a/cli/utils/functions.sh
+++ b/cli/utils/functions.sh
@ -0,0 +1,111 @@
 #!/usr/bin/env bash
 set -euo pipefail
 show_help() {
  cat <<EOF
 ${CLI_NAME}-cli v${CLI_VERSION}
 Manage your Citadel.
 Usage: ${CLI_NAME} <command> [options]
 Flags:
    -h, --help                         Show this help message
    -v, --version                      Show version information for this CLI
 Commands:
    status                             Check the status of all services
    start                              Start the Citadel service
    stop                               Stop the Citadel service safely
    restart                            Restart the Citadel service
    reboot                             Reboot the system
    shutdown                           Shutdown the system
    update                             Update Citadel
    list                               List all installed services apps
    run <service> "<command>"          Run a command inside a container
    set <command>                      Switch between Bitcoin & Lightning implementations
    app <command>                      Install, update or restart apps
    configure <service>                Edit service & app configuration files
    logs <service>                     Show logs for an app or service
    debug                              View logs for troubleshooting
 EOF
 }
 is_managed_by_systemd() {
  if systemctl --all --type service | grep -q "$SERVICE_NAME"; then
    echo true
  else
    echo false
  fi
 }
 is_service_active() {
  service_status=$(systemctl is-active $SERVICE_NAME)
  if [[ "$service_status" = "active" ]]; then
    echo true
  else
    echo false
  fi
 }
 edit_file() {
  if [[ $1 = "--priviledged" ]]; then
    echo "Editing this file requires elevated priviledges."
    if ! sudo test -f $2; then
      echo "File not found."
      exit 1
    fi
    if sudo test -w $2; then
      sudo $EDITOR $2
    else
      echo "File not writable."
    fi
  else
    if ! test -f $1; then
      echo "File not found."
      exit 1
    fi
    if test -w $1; then
      $EDITOR $1
    else
      echo "File not writable."
    fi
  fi
 }
 get_update_channel() {
  update_channel_line=$(cat $CITADEL_ROOT/.env | grep UPDATE_CHANNEL)
  update_channel=(${update_channel_line//=/ })
  if [ -z ${update_channel[1]+x} ]; then
    # fall back to stable
    echo "stable"
  else
    echo ${update_channel[1]}
  fi
 }
 prompt_apply_config() {
  service=$1
  persisted=$2
  read -p "Do you want to apply the changes now? [y/N] " should_restart
  echo
  if [[ $should_restart =~ [Yy]$ ]]; then
    if $persisted; then
      sudo $CITADEL_ROOT/scripts/configure
    fi
    printf "\nRestarting service \"$service\"...\n"
    docker restart $service
    echo "Done."
  else
    if $persisted; then
      echo "To apply the changes, restart Citadel by running \`$CLI_NAME restart\`."
    else
      echo "To apply the changes, restart service \"$service\" by running \`docker restart $service\`."
    fi
  fi
 }
--- a/cli/utils/helpers.sh
+++ b/cli/utils/helpers.sh
@ -0,0 +1,11 @@
 #!/usr/bin/env bash
 set -euo pipefail
 trim() {
  local var="$*"
  # remove leading whitespace characters
  var="${var#"${var%%[![:space:]]*}"}"
  # remove trailing whitespace characters
  var="${var%"${var##*[![:space:]]}"}"
  printf '%s' "$var"
 }
--- a/cli/utils/multiselect.sh
+++ b/cli/utils/multiselect.sh
@ -0,0 +1,123 @@
 #!/usr/bin/env bash
 set -euo pipefail
 function multiselect() {
  # little helpers for terminal print control and key input
  ESC=$(printf "\033")
  cursor_blink_on() { printf "$ESC[?25h"; }
  cursor_blink_off() { printf "$ESC[?25l"; }
  cursor_to() { printf "$ESC[$1;${2:-1}H"; }
  print_inactive() { printf "$2   $1 "; }
  print_active() { printf "$2  $ESC[7m $1 $ESC[27m"; }
  get_cursor_row() {
    IFS=';' read -sdR -p $'\E[6n' ROW COL
    echo ${ROW#*[}
  }
  local return_value=$1
  local -n options=$2
  local -n defaults=$3
  local selected=()
  for ((i = 0; i < ${#options[@]}; i++)); do
    if [[ ${defaults[i]} = "true" ]]; then
      selected+=("true")
    else
      selected+=("false")
    fi
    printf "\n"
  done
  # determine current screen position for overwriting the options
  local lastrow=$(get_cursor_row)
  local startrow=$(($lastrow - ${#options[@]}))
  # ensure cursor and input echoing back on upon a ctrl+c during read -s
  trap "cursor_blink_on; stty echo; printf '\n'; exit" 2
  cursor_blink_off
  key_input() {
    local key
    IFS= read -rsn1 key 2>/dev/null >&2
    if [[ $key = "" ]]; then echo enter; fi
    if [[ $key = $'\x20' ]]; then echo space; fi
    if [[ $key = "k" ]]; then echo up; fi
    if [[ $key = "j" ]]; then echo down; fi
    if [[ $key = $'\x1b' ]]; then
      read -rsn2 key
      if [[ $key = [A || $key = k ]]; then echo up; fi
      if [[ $key = [B || $key = j ]]; then echo down; fi
    fi
  }
  toggle_option() {
    local option=$1
    if [[ ${selected[option]} == true ]]; then
      selected[option]=false
    else
      selected[option]=true
    fi
  }
  print_options() {
    # print options by overwriting the last lines
    local idx=0
    for option in "${options[@]}"; do
      local prefix="[ ]"
      if [[ ${selected[idx]} == true ]]; then
        prefix="[\e[38;5;46m✔\e[0m]"
      fi
      cursor_to $(($startrow + $idx))
      if [ $idx -eq $1 ]; then
        print_active "$option" "$prefix"
      else
        print_inactive "$option" "$prefix"
      fi
      idx=$((idx + 1))
    done
  }
  get_result() {
    local result=()
    for ((i = 0; i < ${#options[@]}; i++)); do
      if ${selected[i]}; then
        result+=(${options[i]})
      fi
    done
    echo "${result[@]}"
  }
  local active=0
  while true; do
    print_options $active
    # user key control
    case $(key_input) in
    space) toggle_option $active ;;
    enter)
      print_options -1
      break
      ;;
    up)
      active=$((active - 1))
      if [ $active -lt 0 ]; then active=$((${#options[@]} - 1)); fi
      ;;
    down)
      active=$((active + 1))
      if [ $active -ge ${#options[@]} ]; then active=0; fi
      ;;
    esac
  done
  # cursor position back to normal
  cursor_to $lastrow
  printf "\n"
  cursor_blink_on
  eval $return_value='("$(get_result)")'
 }
--- a/cli/utils/spinner.sh
+++ b/cli/utils/spinner.sh
@ -0,0 +1,87 @@
 #!/bin/bash
 # Author: Tasos Latsas
 # spinner.sh
 #
 # Display an awesome 'spinner' while running your long shell commands
 #
 # Do *NOT* call _spinner function directly.
 # Use {start,stop}_spinner wrapper functions
 # usage:
 #   1. source this script in your's
 #   2. start the spinner:
 #       start_spinner [display-message-here]
 #   3. run your command
 #   4. stop the spinner:
 #       stop_spinner [your command's exit status]
 #
 # Also see: test.sh
 function _spinner() {
    # $1 start/stop
    #
    # on start: $2 display message
    # on stop : $2 process exit status
    #           $3 spinner function pid (supplied from stop_spinner)
    local on_success="DONE"
    local on_fail="FAIL"
    local white="\e[1;37m"
    local green="\e[1;32m"
    local red="\e[1;31m"
    local nc="\e[0m"
    case $1 in
    start)
        # display message with some space
        echo -ne "${2}   "
        # start spinner
        i=1
        sp='\|/-'
        delay=${SPINNER_DELAY:-0.15}
        while :; do
            printf "\b${sp:i++%${#sp}:1}"
            sleep $delay
        done
        ;;
    stop)
        if [[ -z ${3} ]]; then
            echo "spinner is not running.."
            exit 1
        fi
        kill $3 >/dev/null 2>&1
        # inform the user upon success or failure
        echo -en "\b["
        if [[ $2 -eq 0 ]]; then
            echo -en "${green}${on_success}${nc}"
        else
            echo -en "${red}${on_fail}${nc}"
        fi
        echo -e "]"
        ;;
    *)
        echo "invalid argument, try {start/stop}"
        exit 1
        ;;
    esac
 }
 function start_spinner() {
    # $1 : msg to display
    _spinner "start" "${1}" &
    # set global spinner pid
    _sp_pid=$!
    disown
 }
 function stop_spinner() {
    # $1 : command exit status
    _spinner "stop" $1 $_sp_pid
    unset _sp_pid
 }
--- a/db/dependencies.yml
+++ b/db/dependencies.yml
@ -0,0 +1,5 @@
 compose: v2.6.0
 dashboard: ghcr.io/runcitadel/dashboard:main@sha256:25b6fb413c10f47e186309c8737926c241c0f2bec923b2c08dd837b828f14dbd
 manager: ghcr.io/runcitadel/manager:main@sha256:db5775e986d53e762e43331540bb1c05a27b362da94d587c4a4591c981c00ee4
 middleware: ghcr.io/runcitadel/middleware:main@sha256:2fbbfb2e818bf0462f74a6aaab192881615ae018e6dcb62a50d05f82ec622cb0
 app-cli: ghcr.io/runcitadel/app-cli:main@sha256:f532923eac28cfac03579cbb440397bcf16c8730f291b39eeada8278331f7054
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -2,7 +2,7 @@ version: '3.8'
 services:
  tor:
    container_name: tor
-    image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227
+    image: lncm/tor:0.4.7.8@sha256:aab30ebb496aa25934d6096951d8b200347c3c3ce5db3493695229efa2601f7b
    user: toruser
    restart: on-failure
    volumes:
@ -15,7 +15,7 @@ services:
        ipv4_address: $TOR_PROXY_IP
  app-tor:
    container_name: app-tor
-    image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227
+    image: lncm/tor:0.4.7.8@sha256:aab30ebb496aa25934d6096951d8b200347c3c3ce5db3493695229efa2601f7b
    user: toruser
    restart: on-failure
    volumes:
@ -26,7 +26,7 @@ services:
        ipv4_address: $APPS_TOR_IP
  app-2-tor:
    container_name: app-2-tor
-    image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227
+    image: lncm/tor:0.4.7.8@sha256:aab30ebb496aa25934d6096951d8b200347c3c3ce5db3493695229efa2601f7b
    user: toruser
    restart: on-failure
    volumes:
@ -37,7 +37,7 @@ services:
        ipv4_address: $APPS_2_TOR_IP
  app-3-tor:
    container_name: app-3-tor
-    image: lncm/tor:0.4.7.7@sha256:3c4ae833d2fefbea7d960f833a1e89fc9b2069a6e5f360109b5ddc9334ac0227
+    image: lncm/tor:0.4.7.8@sha256:aab30ebb496aa25934d6096951d8b200347c3c3ce5db3493695229efa2601f7b
    user: toruser
    restart: on-failure
    volumes:
@ -79,7 +79,7 @@ services:
        ipv4_address: $BITCOIN_IP
  lightning:
    container_name: lightning
-    image: lightninglabs/lnd:v0.14.3-beta@sha256:6a2234b0aad4caed3d993736816b198d6228f32c59b27ba2218d5ebf516ae905
+    image: lightninglabs/lnd:v0.15.0-beta@sha256:d227a9db0727ff56020c8d6604c8c369757123d238ab6ce679579c2dd0d0d259
    user: 1000:1000
    depends_on:
    - tor
@ -100,7 +100,7 @@ services:
        ipv4_address: $LND_IP
  dashboard:
    container_name: dashboard
-    image: ghcr.io/runcitadel/dashboard:v0.0.15@sha256:a2cf5ad79367fb083db0f61e5a296aafee655c99af0c228680644c248ec674a5
+    image: ghcr.io/runcitadel/dashboard:main@sha256:25b6fb413c10f47e186309c8737926c241c0f2bec923b2c08dd837b828f14dbd
    restart: on-failure
    stop_grace_period: 1m30s
    networks:
@ -108,7 +108,7 @@ services:
        ipv4_address: $DASHBOARD_IP
  manager:
    container_name: manager
-    image: ghcr.io/runcitadel/manager:v0.0.15@sha256:9fb5a86d9e40a04f93d5b6110d43a0f9a5c4ad6311a843b5442290013196a5ce
+    image: ghcr.io/runcitadel/manager:main@sha256:db5775e986d53e762e43331540bb1c05a27b362da94d587c4a4591c981c00ee4
    depends_on:
    - tor
    - redis
@ -162,7 +162,7 @@ services:
        ipv4_address: $MANAGER_IP
  middleware:
    container_name: middleware
-    image: ghcr.io/runcitadel/middleware:v0.0.11@sha256:e472da8cbfa67d9a9dbf321334fe65cdf20a0f9b6d6bab33fdf07210f54e7002
+    image: ghcr.io/runcitadel/middleware:main@sha256:2fbbfb2e818bf0462f74a6aaab192881615ae018e6dcb62a50d05f82ec622cb0
    depends_on:
    - manager
    - bitcoin
@ -223,6 +223,7 @@ services:
        ipv4_address: $ELECTRUM_IP
  redis:
    container_name: redis
    user: 1000:1000
    image: redis:7.0.0-bullseye@sha256:ad0705f2e2344c4b642449e658ef4669753d6eb70228d46267685045bf932303
    working_dir: /data
    volumes:
@ -234,6 +235,7 @@ services:
    networks:
      default:
        ipv4_address: $REDIS_IP
 networks:
  default:
    name: citadel_main_network
--- a/events/triggers/quick-update
+++ b/events/triggers/quick-update
@ -0,0 +1,25 @@
 #!/usr/bin/env bash
 # SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
 RELEASE=$(cat "$CITADEL_ROOT"/statuses/update-status.json | jq .updateTo -r)
 cat <<EOF > "$CITADEL_ROOT"/statuses/update-status.json
 {"state": "installing", "progress": 30, "description": "Starting update", "updateTo": "$RELEASE"}
 EOF
 curl "https://raw.githubusercontent.com/runcitadel/core/${RELEASE}/db/dependencies.yml" > "$CITADEL_ROOT"/db/dependencies
 cat <<EOF > "$CITADEL_ROOT"/statuses/update-status.json
 {"state": "installing", "progress": 70, "description": "Starting new containers", "updateTo": "$RELEASE"}
 EOF
 "${CITADEL_ROOT}/scripts/start"
 cat <<EOF > "$CITADEL_ROOT"/statuses/update-status.json
 {"state": "success", "progress": 100, "description": "Successfully installed Citadel $RELEASE", "updateTo": ""}
 EOF
--- a/events/triggers/set-update-channel
+++ b/events/triggers/set-update-channel
@ -7,3 +7,4 @@
 CITADEL_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/../..)"
 "${CITADEL_ROOT}/scripts/set-update-channel" "${1}"
 "${CITADEL_ROOT}/scripts/start"
--- a/info.json
+++ b/info.json
@ -1,6 +1,7 @@
 {
-    "version": "0.0.5",
+    "version": "0.1.0-preview.1",
-    "name": "Citadel 0.0.5",
+    "name": "Citadel 0.10 Preview 1",
-    "requires": ">=0.0.1",
+    "requires": ">=0.0.5",
-    "notes": "This update fixes a few bugs in the 0.0.4 release that were preventing some apps from working correctly."
+    "isQuickUpdate": false,
    "notes": "This update brings some of the new features for Citadel 0.1.0 and a lot of internal changes."
 }
--- a/scripts/configure
+++ b/scripts/configure
@ -31,13 +31,14 @@ if not is_arm64 and not is_amd64:
  print('Citadel only works on arm64 and amd64!')
  exit(1)
 dependencies = False
 # Check the output of "docker compose version", if it matches "Docker Compose version v2.0.0-rc.3", return true
 # Otherwise, return false
-def is_compose_rc_or_outdated():
+def is_compose_version_except(target_version):
  try:
    output = subprocess.check_output(['docker', 'compose', 'version'])
-    if output.decode('utf-8').strip() != 'Docker Compose version v2.3.3':
+    if output.decode('utf-8').strip() != 'Docker Compose version {}'.format(target_version):
      print("Using outdated Docker Compose, updating...")
      return True
    else:
      return False
@ -48,17 +49,19 @@ def is_compose_rc_or_outdated():
 def download_docker_compose():
  # Skip if os.path.expanduser('~/.docker/cli-plugins/docker-compose') exists
  subprocess.check_call(["mkdir", "-p", os.path.expanduser('~/.docker/cli-plugins/')])
  if (os.path.exists(os.path.expanduser('~/.docker/cli-plugins/docker-compose')) or os.path.exists('/usr/lib/docker/cli-plugins/docker-compose')) and not is_compose_rc_or_outdated():
    print("Found {}\n".format(subprocess.check_output(['docker', 'compose', 'version']).decode('utf-8').strip()))
    return
  print("Installing Docker Compose...\n")
  if is_arm64:
-    subprocess.check_call(['wget', 'https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-linux-aarch64', '-O', os.path.expanduser('~/.docker/cli-plugins/docker-compose')])
+    compose_arch = 'aarch64'
  elif is_amd64:
-    subprocess.check_call(['wget', 'https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-linux-x86_64', '-O', os.path.expanduser('~/.docker/cli-plugins/docker-compose')])
+    compose_arch = 'x86_64'
-  os.chmod(os.path.expanduser('~/.docker/cli-plugins/docker-compose'), 0o755)
+  # We validate that no other case than the two above can happen before
  if is_compose_version_except(dependencies['compose']):
    print("Docker compose not found or not required version, updating.")
    compose_url = 'https://github.com/docker/compose/releases/download/{}/docker-compose-linux-{}'.format(dependencies['compose'], compose_arch)
    compose_file = os.path.expanduser('~/.docker/cli-plugins/docker-compose')
    subprocess.check_call(['wget', compose_url, '-O', compose_file])
    os.chmod(compose_file, 0o755)
 if not shutil.which("wget"):
    print('Wget is not installed!')
@ -72,6 +75,9 @@ if not shutil.which("docker"):
 CITADEL_ROOT=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 os.chdir(CITADEL_ROOT)
 with open("./db/dependencies.yml", "r") as file: 
  dependencies = yaml.safe_load(file)
 updating = False
 status_dir = os.path.join(CITADEL_ROOT, 'statuses')
 # Make sure to use the main status dir for updates
@ -233,11 +239,7 @@ neutrino.addpeer=testnet2-btcd.zaphq.io
 elif BITCOIN_NETWORK == "signet":
  BITCOIN_RPC_PORT=38332
  BITCOIN_P2P_PORT=38333
-  NEUTRINO_PEERS='''
+  BITCOIN_NODE="bitcoind"
 [neutrino]
 neutrino.addpeer=testnet1-btcd.zaphq.io
 neutrino.addpeer=testnet2-btcd.zaphq.io
  '''
 elif BITCOIN_NETWORK == "regtest":
  BITCOIN_RPC_PORT=18334
  BITCOIN_P2P_PORT=18335
@ -365,6 +367,15 @@ print("Generated configuration files\n")
 print("Checking if Docker Compose is installed...")
 download_docker_compose()
 print("Updating core services...")
 print()
 with open("docker-compose.yml", 'r') as stream:
  compose = yaml.safe_load(stream)
 for service in ["manager", "middleware", "dashboard"]:
  compose["services"][service]["image"] = dependencies[service]
 with open("docker-compose.yml", "w") as stream:
    yaml.dump(compose, stream, sort_keys=False)
 if not reconfiguring:
  print("Updating apps...\n")
  os.system('./scripts/app --invoked-by-configure update')
--- a/scripts/set-update-channel
+++ b/scripts/set-update-channel
@ -10,7 +10,7 @@ NODE_ROOT="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)"
 # If $1 is not given, fail
 if [ -z "$1" ]; then
    echo "Usage: $0 <channel>"
-    echo "Channel can currently either be 'stable' or 'beta'"
+    echo "Channel can currently either be 'stable', 'beta' or 'c-lightning'"
    exit 1
 fi
 sed -i "s/UPDATE_CHANNEL=.*/UPDATE_CHANNEL=${1}/" "${NODE_ROOT}/.env"
--- a/8
+++ b/8
@ -6,10 +6,10 @@
 CITADEL_ROOT="$(dirname $(readlink -f "${BASH_SOURCE[0]}"))"
-alias citadel-update="${CITADEL_ROOT}/scripts/update/update"
+alias citadel="${CITADEL_ROOT}/bin/citadel"
-alias lncli="docker exec -it lnd lncli"
+alias lncli="docker exec -it lightning lncli"
 alias bitcoin-cli="docker exec -it bitcoin bitcoin-cli"
 alias docker-compose="sudo docker compose"
 alias docker="sudo docker"
-alias debug="${CITADEL_ROOT}/scripts/debug"
+
-alias app="${CITADEL_ROOT}/scripts/app"
+export BOS_DEFAULT_LND_PATH="/home/citadel/citadel/lnd"
Author	SHA1	Message	Date
AaronDewes	a13109ee8b	Citadel 0.1.0 Preview 1	2022-07-17 06:54:58 +00:00
Aaron Dewes	d0bc4688d5	Implement quick updates (#56 ) Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com> Co-authored-by: Philipp Walter <philippwalter@pm.me>	2022-07-16 19:28:39 +02:00
Philipp Walter	1ab3c36a12	Feat: add alias for Citadel CLI (#66 ) * feat: add alias for citadel cli * fix: remove redundant aliases	2022-07-16 11:54:22 -05:00
Philipp Walter	b2faa3c287	Merge Stable into Beta (#65 ) * Citadel 0.0.6 (#59) Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com> * feat: add Citadel CLI (#33) add commands for: - Show status for all services - Run a command inside a (app) container - Switch branch / update channel - Switch Bitcoin/Electrum/Lightning implementation - app commands - debug command - Fix update / backup / starting stuck - Edit node configs (Bitcoin Core, LND) - Edit app configs (Nextcloud etc.) - Memory usage / System info - Start - Stop - Restart - Reboot - Shutdown - List all installed apps & services - Logs - Version - Help Co-authored-by: Aaron Dewes <aaron.dewes@protonmail.com> Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com>	2022-07-15 18:34:49 +02:00
Philipp Walter	d70f727a91	Small fixes (#63 ) * Citadel 0.0.6 (#59) Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com> * fix: lncli alias * fix: use bitcoind for signet Co-authored-by: Aaron Dewes <aaron.dewes@protonmail.com> Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com>	2022-07-15 13:29:22 +02:00
Aaron Dewes	6e9ecd85ae	Revert "[Feat] Docker compose isolation (#62 )" (#64 ) This reverts commit `1b61d525f6`.	2022-07-14 20:35:59 -04:00
Lele	1b61d525f6	[Feat] Docker compose isolation (#62 ) * Moved docker-compose.yml out of default place * Changed docker-compose.yml references everywhere, added as default option * Fixed docker-compose name to have yaml extension * Fixed docker-compose usage with new name * Removed Docker cmd from setenv	2022-07-13 13:43:48 +02:00
Aaron Dewes	17c116e1ac	Add support for BOS (#61 ) This makes it easier to install BOS on the host. The setenv script is run on every shell open on every Citadel OS install by default.	2022-07-09 21:55:07 -04:00
nolim1t - f6287b82CC84bcbd	55078c5679	Update LND to 0.15.0 (#60 ) Co-authored-by: nolim1t - f6287b82CC84bcbd <nolim1t@users.noreply.github.com> Co-authored-by: Aaron Dewes <aaron.dewes@protonmail.com>	2022-07-05 14:56:17 +02:00
nolim1t - f6287b82CC84bcbd	c4de7d10aa	Update Tor to 0.4.7.8 (#58 )	2022-06-20 15:55:27 +02:00
Aaron Dewes	faf6d62e42	App system cleanups (#51 ) * Add app cli to docker-compose.yml * Remove app.yml v1 * Add missing import * More cleanups * Another missing import * Add mount for apps * Remove more	2022-06-18 20:05:22 -04:00
AaronDewes	ceb77e8bdd	Merge tag 'v0.0.5' into beta	2022-06-11 16:45:12 +00:00
Aaron Dewes	7311369fc8	Bump version	2022-06-08 07:47:29 +02:00
Aaron Dewes	b87c54e219	Fix memory scripts (#50 ) This also reduces the script interval, because it is still very slow.	2022-06-07 14:33:14 -04:00
Aaron Dewes	2a933eaa1b	Fix the way UDP ports are declared (#49 ) Previously, the way they had been defined was invalid	2022-06-07 14:28:45 -04:00
Aaron Dewes	e40e35ffb2	Update manager (#48 )	2022-06-05 22:06:32 +02:00
Aaron Dewes	f08711ae7c	Add script to set update channel (#47 )	2022-06-05 21:03:29 +02:00
Aaron Dewes	02fdae4971	Publish a beta of 0.0.5 to the beta channel (#46 )	2022-06-05 20:07:01 +02:00
Aaron Dewes	ce71560ef0	New backup server (#43 )	2022-06-05 10:03:26 +02:00
Aaron Dewes	86c17c365e	Remove leftovers from karen v1 (#42 )	2022-06-01 21:14:49 -04:00
AaronDewes	5b2b5a4541	Merge branch 'stable' into beta	2022-06-01 16:44:19 +00:00
AaronDewes	6e74290691	Merge branch 'stable' into beta	2022-05-24 05:39:59 +00:00
AaronDewes	62d51aa807	0.0.4 rc1	2022-05-22 14:34:09 +00:00
AaronDewes	781299fa1a	Update manager	2022-05-22 13:10:26 +00:00
AaronDewes	4eb9819cf9	Remove leftovers from WIP karen v2	2022-05-22 13:08:07 +00:00
AaronDewes	9a6501a80e	Merge remote-tracking branch 'origin/karen-v2' into beta	2022-05-22 13:07:12 +00:00
AaronDewes	d45da547d6	Merge branch 'stable' into beta	2022-05-22 13:06:30 +00:00
Aaron Dewes	80ead94dbd	Remove unused import	2022-05-20 10:12:53 +02:00
Aaron Dewes	a5e74fa1d2	Remove log	2022-05-19 19:03:55 +02:00
AaronDewes	f6dae9c646	Add karen v2	2022-05-19 10:41:21 +00:00
AaronDewes	3cec30340b	Generate cache files for compose generator	2022-05-16 09:10:52 +00:00
AaronDewes	611d4a166b	Add Karen v2 demo	2022-05-16 07:19:50 +00:00
AaronDewes	56946a9812	Put the latest beta into info.json	2022-05-16 06:11:47 +00:00
AaronDewes	6b29a76d81	Switch app store to beta branch	2022-05-16 06:11:07 +00:00
		`@ -1,3 +0,0 @@`
			`SPDX-FileCopyrightText: 2021 Citadel and contributors`

			`SPDX-License-Identifier: GPL-3.0-or-later`