App.yml v3 (draft)

2024-12-28 15:42:59 +00:00 · 2022-04-07 13:24:48 +00:00 · 2022-04-07 13:24:48 +00:00 · a600fd4946
commit a600fd4946
parent 4b8c42bedd
10 changed files with 748 additions and 91 deletions
--- a/app/app-standard-v3.yml
+++ b/app/app-standard-v3.yml
@ -0,0 +1,201 @@
+# yaml-language-server: $schema=https://json-schema.org/draft/2020-12/schema
+$schema: https://json-schema.org/draft/2020-12/schema
+
+
+title: Citadel app.yml v3
+description: The third revision of Citadel's app.yml format
+type: object
+
+properties:
+  version:
+    type:
+      - string
+      - number
+    description: The version of the app.yml format you're using.
+
+  metadata:
+    type: object
+    properties:
+      name:
+        description: Displayed name of the app
+        type: string
+      version:
+        description: Displayed version for the app
+        type: string
+      category:
+        description: The category you'd put the app in
+        type: string
+      tagline:
+        description: A clever tagline
+        type: string
+      description:
+        description: A longer description of the app
+        type: string
+      developer:
+        description: The awesome people behind the app
+        type: string
+      website:
+        description: Displayed version for the app
+        type: string
+      dependencies:
+        description: >-
+          The services the app depends on.
+
+          This can also contain an array like [c-lightning, lnd] if your app requires one of two dependencies to function.
+        type: array
+        items:
+          type: [string, array]
+          items:
+            type: string
+      repo:
+        description: The development repository for your app
+        type: string
+      support:
+        description: A link to the app support wiki/chat/...
+        type: string
+      gallery:
+        type: array
+        description: >-
+          URLs or paths in the runcitadel/app-images/[app-name] folder with app
+          images
+        items:
+          type: string
+      path:
+        description: The path of the app's visible site the open button should open
+        type: string
+      defaultPassword:
+        description: >-
+          The app's default password.
+          Set this to $APP_SEED if the password is the environment variable $APP_SEED.
+        type: string
+      torOnly:
+        description: Whether the app is only available over tor
+        type: boolean
+      updateContainer:
+        type:
+          - string
+          - array
+        description: The container(s) the developer system should automatically update.
+    required:
+      - name
+      - version
+      - category
+      - tagline
+      - description
+      - developer
+      - website
+      - repo
+      - support
+      - gallery
+    additionalProperties: false
+
+  containers:
+    type: array
+    items:
+      type: object
+      properties:
+        name:
+          type: string
+        image:
+          type: string
+        requiredPorts:
+          type: array
+          items:
+            type: number
+          description: Ports this container requires to be exposed to work properly
+        port:
+          type: number
+          description: >-
+            If this is the main container, the port inside the container which
+            will be exposed to the outside as the port specified in metadata.
+            If this is not set, the port is passed as an env variable in the format APP_${APP_NAME}_${CONTAINER_NAME}_PORT
+        preferredOutsidePort:
+          type: number
+          description: The port this container would like to have "port" exposed as.
+        requiresPort:
+          type: boolean
+          description: Set this to true if the app requires the preferredOutsidePort to be the real outside port.
+        environment:
+          type: object
+        data:
+          type: array
+          description: >-
+            An array of at directories in the container the app stores its data
+            in. Can be empty. Please only list top-level directories.
+          items:
+            type: string
+        user:
+          type: string
+          description: The user the container should run as
+        stop_grace_period:
+          type: string
+          description: The grace period for stopping the container. Defaults to 1 minute.
+        depends_on:
+          type: array
+          description: The services the container depends on
+        entrypoint:
+          type:
+            - string
+            - array
+          description: The entrypoint for the container
+        mounts:
+          type: object
+          description: Where to mount some services' data directories
+          properties:
+            bitcoin:
+              type: string
+              description: Where to mount the bitcoin dir
+            lnd:
+              type: string
+              description: Where to mount the lnd dir
+            c_lightning:
+              type: string
+              description: Where to mount the c-lightning dir
+            additionalProperties: false
+        command:
+          type:
+            - string
+            - array
+          description: The command for the container
+        init:
+          type: boolean
+          description: Whether the container should be run with init
+        stop_signal:
+          type: string
+          description: The signal to send to the container when stopping
+        noNetwork:
+          type: boolean
+          description: >-
+            Set this to true if the container shouldn't get an IP & port
+            exposed. This isn't necessary, but helps the docker-compose.yml generator to generate a cleaner output.
+        hiddenServicePorts:
+          type: 
+            - object
+            - number
+            - array
+          items:
+            type:
+              - string
+              - number
+              - array
+          description: >-
+            This can either be a map of hidden service names (human readable names, not the .onion URL, and strings, not numbers)
+            to a port if your app needs multiple hidden services on different ports,
+            a map of port inside to port on the hidden service (if your app has multiple ports on one hidden service),
+            or simply one port number if your apps hidden service should only expose one port to the outside which isn't 80.
+        restart:
+          type: string
+          description: When the container should restart. Can be 'always' or 'on-failure'.
+        network_mode:
+          type: string
+      additionalProperties: false
+      required:
+        - name
+        - image
+    additionalProperties: false
+
+required:
+  - metadata
+  - containers
+
+additionalProperties: false
--- a/app/lib/composegenerator/v1/types.py
+++ b/app/lib/composegenerator/v1/types.py
@ -1,4 +1,4 @@
-from typing import Union
+from typing import Union, List
 from dataclasses import dataclass, field
 from dacite import from_dict

@ -14,8 +14,8 @@ class Metadata:
    website: str
    repo: str
    support: str
-    gallery: list[str] = field(default_factory=list)
-    dependencies: list[str] = field(default_factory=list)
+    gallery: List[str] = field(default_factory=list)
+    dependencies: List[str] = field(default_factory=list)
    mainContainer: Union[str, None] = None
    updateContainer: Union[str, None] = None
    path: str = ""
@ -34,9 +34,9 @@ class Container:
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
    depends_on: list = field(default_factory=list)
-    entrypoint: Union[list[str], str] = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
    bitcoin_mount_dir: Union[str, None] = None
-    command: Union[list[str], str] = field(default_factory=list)
+    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
@ -52,7 +52,7 @@ class Container:
 class App:
    version: Union[str, int]
    metadata: Metadata
-    containers: list[Container]
+    containers: List[Container]

 # Generate an app instance from an app dict
 def generateApp(appDict):
@ -72,21 +72,21 @@ class ContainerStage2:
    id: str
    name: str
    image: str
-    permissions: list[str] = field(default_factory=list)
+    permissions: List[str] = field(default_factory=list)
    ports: list = field(default_factory=list)
    environment: Union[dict, None] = None
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
-    depends_on: list[str] = field(default_factory=list)
-    entrypoint: Union[list[str], str] = field(default_factory=list)
-    command: Union[list[str], str] = field(default_factory=list)
+    depends_on: List[str] = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
+    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
    needsHiddenService: Union[bool, None] = None
    hiddenServicePort: Union[int, None] = None
    hiddenServicePorts: Union[dict, None] = None
-    volumes: list[str] = field(default_factory=list)
+    volumes: List[str] = field(default_factory=list)
    networks: NetworkConfig = field(default_factory=NetworkConfig)
    restart: Union[str, None] = None

@ -94,7 +94,7 @@ class ContainerStage2:
 class AppStage2:
    version: Union[str, int]
    metadata: Metadata
-    containers: list[ContainerStage2]
+    containers: List[ContainerStage2]

@dataclass
 class MetadataStage3:
@ -106,10 +106,10 @@ class MetadataStage3:
    description: str
    developer: str
    website: str
-    dependencies: list[str]
+    dependencies: List[str]
    repo: str
    support: str
-    gallery: list[str]
+    gallery: List[str]
    mainContainer: Union[str, None] = None
    updateContainer: Union[str, None] = None
    path: str = ""
@ -120,7 +120,7 @@ class MetadataStage3:
 class AppStage3:
    version: Union[str, int]
    metadata: MetadataStage3
-    containers: list[ContainerStage2]
+    containers: List[ContainerStage2]

@dataclass
 class ContainerStage4:
@ -131,16 +131,16 @@ class ContainerStage4:
    environment: Union[dict, None] = None
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
-    depends_on: list[str] = field(default_factory=list)
-    entrypoint: Union[list[str], str] = field(default_factory=list)
-    command: Union[list[str], str] = field(default_factory=list)
+    depends_on: List[str] = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
+    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
    needsHiddenService: Union[bool, None] = None
    hiddenServicePort: Union[int, None] = None
    hiddenServicePorts: Union[dict, None] = None
-    volumes: list[str] = field(default_factory=list)
+    volumes: List[str] = field(default_factory=list)
    networks: NetworkConfig = field(default_factory=NetworkConfig)
    restart: Union[str, None] = None

@ -148,4 +148,4 @@ class ContainerStage4:
 class AppStage4:
    version: Union[str, int]
    metadata: MetadataStage3
-    services: list[ContainerStage4]
+    services: List[ContainerStage4]
--- a/app/lib/composegenerator/v1/utils/networking.py
+++ b/app/lib/composegenerator/v1/utils/networking.py
@ -8,10 +8,8 @@ import random

 from lib.composegenerator.v1.types import Container

-
 def getFreePort(networkingFile: str, appId: str):
    # Ports used currently in Citadel
-    # TODO: Update this list, currently it's outdated
    usedPorts = [
        # Dashboard
        80,
@ -29,26 +27,6 @@ def getFreePort(networkingFile: str, appId: str):
        50001,
        # Tor Proxy
        9050,
-        # Soon hardcoded to Specter
-        25441,
-        3003,
-        3007,
-        3006,
-        3009,
-        3005,
-        8898,
-        3008,
-        8081,
-        8082,
-        8083,
-        8085,
-        2222,
-        8086,
-        8087,
-        8008,
-        8088,
-        8089,
-        8091
    ]
    networkingData = {}
    if os.path.isfile(networkingFile):
--- a/app/lib/composegenerator/v2/types.py
+++ b/app/lib/composegenerator/v2/types.py
@ -1,4 +1,4 @@
-from typing import Union
+from typing import Union, List
 from dataclasses import dataclass, field
 from dacite import from_dict

@ -14,8 +14,8 @@ class Metadata:
    website: str
    repo: str
    support: str
-    gallery: list[str] = field(default_factory=list)
-    dependencies: list[str] = field(default_factory=list)
+    gallery: List[str] = field(default_factory=list)
+    dependencies: List[str] = field(default_factory=list)
    updateContainer: Union[str, Union[list, None]] = field(default_factory=list)
    path: str = ""
    defaultPassword: str = ""
@ -37,11 +37,11 @@ class Container:
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
    depends_on: list = field(default_factory=list)
-    entrypoint: Union[list[str], str] = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
    bitcoin_mount_dir: Union[str, None] = None
    lnd_mount_dir: Union[str, None] = None
    c_lightning_mount_dir: Union[str, None] = None
-    command: Union[list[str], str] = field(default_factory=list)
+    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
@ -56,7 +56,7 @@ class Container:
 class App:
    version: Union[str, int]
    metadata: Metadata
-    containers: list[Container]
+    containers: List[Container]

 # Generate an app instance from an app dict
 def generateApp(appDict):
@ -76,19 +76,19 @@ class ContainerStage2:
    id: str
    name: str
    image: str
-    permissions: list[str] = field(default_factory=list)
+    permissions: List[str] = field(default_factory=list)
    ports: list = field(default_factory=list)
    environment: Union[dict, None] = None
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
-    depends_on: list[str] = field(default_factory=list)
-    entrypoint: Union[list[str], str] = field(default_factory=list)
-    command: Union[list[str], str] = field(default_factory=list)
+    depends_on: List[str] = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
+    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
-    volumes: list[str] = field(default_factory=list)
+    volumes: List[str] = field(default_factory=list)
    networks: NetworkConfig = field(default_factory=NetworkConfig)
    restart: Union[str, None] = None
    network_mode: Union[str, None] = None
@ -97,7 +97,7 @@ class ContainerStage2:
 class AppStage2:
    version: Union[str, int]
    metadata: Metadata
-    containers: list[ContainerStage2]
+    containers: List[ContainerStage2]

@dataclass
 class MetadataStage3:
@ -109,10 +109,10 @@ class MetadataStage3:
    description: str
    developer: str
    website: str
-    dependencies: list[str]
+    dependencies: List[str]
    repo: str
    support: str
-    gallery: list[str]
+    gallery: List[str]
    updateContainer: Union[str, Union[list, None]] = field(default_factory=list)
    path: str = ""
    defaultPassword: str = ""
@ -126,7 +126,7 @@ class MetadataStage3:
 class AppStage3:
    version: Union[str, int]
    metadata: MetadataStage3
-    containers: list[ContainerStage2]
+    containers: List[ContainerStage2]

@dataclass
 class ContainerStage4:
@ -137,14 +137,14 @@ class ContainerStage4:
    environment: Union[dict, None] = None
    user: Union[str, None] = None
    stop_grace_period: str = '1m'
-    depends_on: list[str] = field(default_factory=list)
-    entrypoint: Union[list[str], str] = field(default_factory=list)
-    command: Union[list[str], str] = field(default_factory=list)
+    depends_on: List[str] = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
+    command: Union[List[str], str] = field(default_factory=list)
    init: Union[bool, None] = None
    stop_signal: Union[str, None] = None
    noNetwork: Union[bool, None] = None
    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
-    volumes: list[str] = field(default_factory=list)
+    volumes: List[str] = field(default_factory=list)
    networks: NetworkConfig = field(default_factory=NetworkConfig)
    restart: Union[str, None] = None
    network_mode: Union[str, None] = None
@ -153,4 +153,4 @@ class ContainerStage4:
 class AppStage4:
    version: Union[str, int]
    metadata: MetadataStage3
-    services: list[ContainerStage4]
+    services: List[ContainerStage4]
--- a/app/lib/composegenerator/v3/generate.py
+++ b/app/lib/composegenerator/v3/generate.py
@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+
+from lib.citadelutils import classToDict
+from lib.composegenerator.shared.main import convertDataDirToVolume, convertContainersToServices
+from lib.composegenerator.shared.env import validateEnv
+from lib.composegenerator.v2.networking import configureIps, configureHiddenServices
+
+from lib.composegenerator.v3.types import App, AppStage2, AppStage4, generateApp
+from lib.composegenerator.v3.networking import configureMainPort
+from lib.composegenerator.shared.const import permissions
+
+
+def convertContainerPermissions(app: App) -> App:
+    for container in app.containers:
+        for permission in app.metadata.dependencies:
+            if permission in permissions():
+                container.environment_allow.extend(permissions()[permission]['environment_allow'])
+                container.volumes.extend(permissions()[permission]['volumes'])
+            else:
+                print("Warning: app {} defines unknown permission {}".format(container.name, app.metadata.name, permission))
+    return app
+
+def convertDataDirToVolumeGen3(app: App) -> AppStage2:
+    for container in app.containers:
+        # Loop through data dirs in container.data, if they don't contain a .., add them to container.volumes
+        # Also, a datadir shouldn't start with a /
+        for dataDir in container.data:
+            if dataDir.find("..") == -1 and dataDir[0] != "/":
+                container.volumes.append(
+                    '${APP_DATA_DIR}/' + dataDir)
+            else:
+                print("Data dir " + dataDir +
+                        " contains invalid characters")
+        del container.data
+    for container in app.containers:
+        if container.mounts:
+            if container.mounts.lnd:
+                if not 'lnd' in app.metadata.dependencies:
+                    print("Warning: container {} of app {} defines lnd mount dir but doesn't request lnd permission".format(container.name, app.metadata.name))
+                    # Skip this container
+                    continue
+                # Also skip the container if container.mounts.lnd contains a :
+                if container.mounts.lnd.find(":") == -1:
+                    container.volumes.append('${LND_DATA_DIR}:' + container.mounts.lnd)
+            if container.mounts.bitcoin:
+                if not 'bitcoind' in app.metadata.dependencies:
+                    print("Warning: container {} of app {} defines lnd mount dir but doesn't request lnd permission".format(container.name, app.metadata.name))
+                    # Skip this container
+                    continue
+                # Also skip the container if container.lnd_mount_dir contains a :
+                if container.mounts.bitcoin.find(":") == -1:
+                    container.volumes.append('${LND_DATA_DIR}:' + container.mounts.bitcoin)
+            del container.mounts
+                
+    return app
+
+def createComposeConfigFromV3(app: dict, nodeRoot: str):
+    envFile = os.path.join(nodeRoot, ".env")
+    networkingFile = os.path.join(nodeRoot, "apps", "networking.json")
+
+    newApp: App = generateApp(app)
+    newApp = convertContainerPermissions(newApp)
+    newApp = validateEnv(newApp)
+    newApp = convertDataDirToVolumeGen3(newApp)
+    newApp = configureIps(newApp, networkingFile, envFile)
+    newApp = configureMainPort(newApp, nodeRoot)
+    # This is validated earlier
+    for container in newApp.containers:
+        container.ports = container.requiredPorts
+        del container.requiredPorts
+    newApp = configureHiddenServices(newApp, nodeRoot)
+    finalConfig: AppStage4 = convertContainersToServices(newApp)
+    newApp = classToDict(finalConfig)
+    del newApp['metadata']
+    if "version" in newApp:
+        del newApp["version"]
+    # Set version to 3.8 (current compose file version)
+    newApp = {'version': '3.8', **newApp}
+    return newApp
--- a/app/lib/composegenerator/v3/networking.py
+++ b/app/lib/composegenerator/v3/networking.py
@ -0,0 +1,102 @@
+# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from lib.composegenerator.v3.types import App, AppStage2, AppStage3, Container
+from lib.citadelutils import parse_dotenv
+import json
+from os import path
+import random
+from lib.composegenerator.v1.networking import assignIp, assignPort
+
+
+def getMainContainer(app: App) -> Container:
+    if len(app.containers) == 1:
+        return app.containers[0]
+    else:
+        for container in app.containers:
+            # Main is recommended, support web for easier porting from Umbrel
+            if container.name == 'main' or container.name == 'web':
+                return container
+    # Fallback to first container
+    return app.containers[0]
+
+
+def configureMainPort(app: AppStage2, nodeRoot: str) -> AppStage3:
+    registryFile = path.join(nodeRoot, "apps", "registry.json")
+    portsFile = path.join(nodeRoot, "apps", "ports.json")
+    envFile = path.join(nodeRoot, ".env")
+    registry: list = []
+    ports = {}
+    if path.isfile(registryFile):
+        with open(registryFile, 'r') as f:
+            registry = json.load(f)
+    else:
+        raise Exception("Registry file not found")
+    if path.isfile(portsFile):
+        with open(portsFile, 'r') as f:
+            ports = json.load(f)
+    else:
+        raise Exception("Ports file not found")
+
+    mainContainer = getMainContainer(app)
+
+    portAsEnvVar = "APP_{}_{}_PORT".format(
+        app.metadata.id.upper().replace("-", "_"),
+        mainContainer.name.upper().replace("-", "_")
+    )
+    portToAppend = portAsEnvVar
+
+    mainPort = False
+
+    if mainContainer.port:
+        portToAppend = "${{{}}}:{}".format(portAsEnvVar, mainContainer.port)
+        mainPort = mainContainer.port
+        for port in ports[app.metadata.id][mainContainer.name]:
+            if str(port["internalPort"]) == str(mainPort):
+                containerPort = port["publicPort"]
+        del mainContainer.port
+    elif not mainContainer.requiredPorts:
+        for port in ports[app.metadata.id][mainContainer.name]:
+            if port["dynamic"]:
+                mainPort = port["internalPort"]
+                containerPort = port["publicPort"]
+        portToAppend = "${{{}}}:${{{}}}".format(portAsEnvVar, portAsEnvVar)
+
+    if mainContainer.ports:
+        mainContainer.ports.append(portToAppend)
+        # Set the main port to the first port in the list, if it contains a :, it's the port after the :
+        # If it doesn't contain a :, it's the port itself
+        if mainPort == False:
+            mainPort = mainContainer.ports[0]
+            if mainPort.find(":") != -1:
+                mainPort = mainPort.split(":")[1]
+    else:
+        mainContainer.ports = [portToAppend]
+        if mainPort == False:
+            mainPort = portDetails['port']
+
+    if mainContainer.network_mode != "host":
+        mainContainer = assignIp(mainContainer, app.metadata.id, path.join(
+            nodeRoot, "apps", "networking.json"), path.join(nodeRoot, ".env"))
+
+    # Also set the port in metadata
+    app.metadata.port = int(containerPort)
+    if mainPort:
+        app.metadata.internalPort = int(mainPort)
+    else:
+        app.metadata.internalPort = int(containerPort)
+
+    for registryApp in registry:
+        if registryApp['id'] == app.metadata.id:
+            registry[registry.index(registryApp)]['port'] = int(containerPort)
+            registry[registry.index(registryApp)]['internalPort'] = app.metadata.internalPort
+            break
+
+    with open(registryFile, 'w') as f:
+        json.dump(registry, f, indent=4, sort_keys=True)
+
+    with open(envFile, 'a') as f:
+        f.write("{}={}\n".format(portAsEnvVar, app.metadata.port))
+    return app
+
--- a/app/lib/composegenerator/v3/types.py
+++ b/app/lib/composegenerator/v3/types.py
@ -0,0 +1,163 @@
+from typing import Union, List
+from dataclasses import dataclass, field
+from dacite import from_dict
+
+@dataclass
+class Metadata:
+    id: str
+    name: str
+    version: str
+    category: str
+    tagline: str
+    description: str
+    developer: str
+    website: str
+    repo: str
+    support: str
+    gallery: List[Union[list,str]] = field(default_factory=list)
+    dependencies: List[str] = field(default_factory=list)
+    updateContainer: Union[str, Union[list, None]] = field(default_factory=list)
+    path: str = ""
+    defaultPassword: str = ""
+    torOnly: bool = False
+    lightningImplementation: Union[str, None] = None
+    # Added automatically later
+    port: int = 0
+    internalPort: int = 0
+
+@dataclass
+class ContainerMounts:
+    bitcoin: Union[str, None] = None
+    lnd: Union[str, None] = None
+    c_lightning: Union[str, None] = None
+
+@dataclass
+class Container:
+    name: str
+    image: str
+    permissions: list = field(default_factory=list)
+    port: Union[int, None] = None
+    requiredPorts: list = field(default_factory=list)
+    preferredOutsidePort: list = field(default_factory=list)
+    requiresPort: Union[bool, None] = None
+    environment: Union[dict, None] = None
+    data: list = field(default_factory=list)
+    user: Union[str, None] = None
+    stop_grace_period: str = '1m'
+    depends_on: list = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
+    mounts: Union[ContainerMounts, None] = None
+    command: Union[List[str], str] = field(default_factory=list)
+    init: Union[bool, None] = None
+    stop_signal: Union[str, None] = None
+    noNetwork: Union[bool, None] = None
+    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
+    environment_allow: list = field(default_factory=list)
+    network_mode: Union[str, None] = None
+    # Only added later
+    volumes: list = field(default_factory=list)
+    restart: Union[str, None] = None
+    ports: list = field(default_factory=list)
+
+@dataclass
+class App:
+    version: Union[str, int]
+    metadata: Metadata
+    containers: List[Container]
+
+# Generate an app instance from an app dict
+def generateApp(appDict):
+    return from_dict(data_class=App, data=appDict)
+
+@dataclass
+class Network:
+    ipv4_address: Union[str, None] = None
+
+@dataclass
+class NetworkConfig:
+    default: Network
+
+# After converting data dir and defining volumes, stage 2
+@dataclass
+class ContainerStage2:
+    id: str
+    name: str
+    image: str
+    permissions: List[str] = field(default_factory=list)
+    ports: list = field(default_factory=list)
+    environment: Union[dict, None] = None
+    user: Union[str, None] = None
+    stop_grace_period: str = '1m'
+    depends_on: List[str] = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
+    command: Union[List[str], str] = field(default_factory=list)
+    init: Union[bool, None] = None
+    stop_signal: Union[str, None] = None
+    noNetwork: Union[bool, None] = None
+    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
+    volumes: List[str] = field(default_factory=list)
+    networks: NetworkConfig = field(default_factory=NetworkConfig)
+    restart: Union[str, None] = None
+    network_mode: Union[str, None] = None
+
+@dataclass
+class AppStage2:
+    version: Union[str, int]
+    metadata: Metadata
+    containers: List[ContainerStage2]
+
+@dataclass
+class MetadataStage3:
+    id: str
+    name: str
+    version: str
+    category: str
+    tagline: str
+    description: str
+    developer: str
+    website: str
+    dependencies: List[str]
+    repo: str
+    support: str
+    gallery: List[str]
+    updateContainer: Union[str, Union[list, None]] = field(default_factory=list)
+    path: str = ""
+    defaultPassword: str = ""
+    torOnly: bool = False
+    lightningImplementation: Union[str, None] = None
+    # Added automatically later
+    port: int = 0
+    internalPort: int = 0
+
+@dataclass
+class AppStage3:
+    version: Union[str, int]
+    metadata: MetadataStage3
+    containers: List[ContainerStage2]
+
+@dataclass
+class ContainerStage4:
+    id: str
+    name: str
+    image: str
+    ports: list = field(default_factory=list)
+    environment: Union[dict, None] = None
+    user: Union[str, None] = None
+    stop_grace_period: str = '1m'
+    depends_on: List[str] = field(default_factory=list)
+    entrypoint: Union[List[str], str] = field(default_factory=list)
+    command: Union[List[str], str] = field(default_factory=list)
+    init: Union[bool, None] = None
+    stop_signal: Union[str, None] = None
+    noNetwork: Union[bool, None] = None
+    hiddenServicePorts: Union[dict, Union[int, Union[None, list]]] = field(default_factory=list)
+    volumes: List[str] = field(default_factory=list)
+    networks: NetworkConfig = field(default_factory=NetworkConfig)
+    restart: Union[str, None] = None
+    network_mode: Union[str, None] = None
+
+@dataclass
+class AppStage4:
+    version: Union[str, int]
+    metadata: MetadataStage3
+    services: List[ContainerStage4]
--- a/app/lib/manage.py
+++ b/app/lib/manage.py
@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 # SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
@ -24,11 +22,12 @@ except Exception:
    print("    sudo apt install -y python3-semver")
    print("On other systems, please use")
    print("     sudo pip3 install semver")
-    print("Continuing anyway, but some features won't be available")
-    print("Like checking for app updates")
+    print("Continuing anyway, but some features won't be available,")
+    print("for example checking for app updates")

 from lib.composegenerator.v1.generate import createComposeConfigFromV1
 from lib.composegenerator.v2.generate import createComposeConfigFromV2
+from lib.composegenerator.v3.generate import createComposeConfigFromV3
 from lib.validate import findAndValidateApps
 from lib.metadata import getAppRegistry
 from lib.entropy import deriveEntropy
@ -85,7 +84,9 @@ def update(verbose: bool = False):
    # The compose generation process updates the registry, so we need to get it set up with the basics before that
    registry = getAppRegistry(apps, appsDir)
    with open(os.path.join(appsDir, "registry.json"), "w") as f:
-        json.dump(registry, f, sort_keys=True)
+        json.dump(registry["metadata"], f, sort_keys=True)
+    with open(os.path.join(appsDir, "ports.json"), "w") as f:
+        json.dump(registry["ports"], f, sort_keys=True)
    print("Wrote registry to registry.json")

    # Loop through the apps and generate valid compose files from them, then put these into the app dir
@ -188,9 +189,13 @@ def getApp(appFile: str, appId: str):
    app["metadata"]["id"] = appId

    if 'version' in app and str(app['version']) == "1":
+        print("Warning: App {} uses version 1 of the app.yml format, which is scheduled for removal in Citadel 0.1.0".format(appId))
        return createComposeConfigFromV1(app, nodeRoot)
    elif 'version' in app and str(app['version']) == "2":
+        print("Warning: App {} uses version 2 of the app.yml format, which is scheduled for removal in Citadel 0.2.0".format(appId))
        return createComposeConfigFromV2(app, nodeRoot)
+    elif 'version' in app and str(app['version']) == "3":
+        return createComposeConfigFromV3(app, nodeRoot)
    else:
        raise Exception("Error: Unsupported version of app.yml")

@ -341,8 +346,8 @@ def updateRepos():
        subprocess.run("git clone --depth 1 --branch {} {} {}".format(branch, gitUrl, tempDir), shell=True, stdout=subprocess.DEVNULL)
        # Overwrite the current app dir with the contents of the temporary dir/apps/app
        for app in os.listdir(os.path.join(tempDir, "apps")):
-            # if the app is already installed, don't overwrite it
-            if app in alreadyInstalled:
+            # if the app is already installed (or a simple file instead of a valid app), skip it
+            if app in alreadyInstalled or not os.path.isdir(os.path.join(tempDir, "apps", app)):
                continue
            if gitUrl.startswith("https://github.com"):
                sourceMap[app] = {
--- a/app/lib/metadata.py
+++ b/app/lib/metadata.py
@ -5,19 +5,32 @@
 import os
 import yaml

-from lib.composegenerator.v1.networking import getMainContainer
+from lib.composegenerator.v2.networking import getMainContainer
+from lib.composegenerator.v1.networking import getFreePort
 from lib.entropy import deriveEntropy
+from typing import List
+import json
+import random

-def getUpdateContainer(app: dict):
-    if len(app['containers']) == 1:
-        return app['containers'][0]
-    else:
-        if 'updateContainer' in app['metadata']:
-            for container in app['containers']:
-                    if container['name'] == app['metadata']['updateContainer']:
-                        return container
-    return getMainContainer(app)
+appPorts = {}
+appPortMap = {}
+disabledApps = []

+def appPortsToMap():
+    for port in appPorts:
+        appId = appPorts[port]["app"]
+        containerId = appPorts[port]["container"]
+        realPort = appPorts[port]["port"]
+        if not appId in appPortMap:
+            appPortMap[appId] = {}
+        if not containerId in appPortMap[appId]:
+            appPortMap[appId][containerId] = []
+        appPortMap[appId][containerId].append({
+            "publicPort": port,
+            "internalPort": realPort,
+            "dynamic": appPorts[port]["dynamic"]
+        })
+        
 # For every app, parse the app.yml in ../apps/[name] and
 # check their metadata, and return a list of all app's metadata
 # Also check the path and defaultPassword and set them to an empty string if they don't exist
@ -28,15 +41,117 @@ def getAppRegistry(apps, app_path):
    for app in apps:
        app_yml_path = os.path.join(app_path, app, 'app.yml')
        if os.path.isfile(app_yml_path):
-            with open(app_yml_path, 'r') as f:
-                app_yml = yaml.safe_load(f.read())
-            metadata: dict = app_yml['metadata']
-            metadata['id'] = app
-            metadata['path'] = metadata.get('path', '')
-            metadata['defaultPassword'] = metadata.get('defaultPassword', '')
-            if metadata['defaultPassword'] == "$APP_SEED":
-                metadata['defaultPassword'] = deriveEntropy("app-{}-seed".format(app))
-            if "mainContainer" in metadata:
-                metadata.pop("mainContainer")
-            app_metadata.append(metadata)
-    return app_metadata
+            try:
+                with open(app_yml_path, 'r') as f:
+                    app_yml = yaml.safe_load(f.read())
+                metadata: dict = app_yml['metadata']
+                metadata['id'] = app
+                metadata['path'] = metadata.get('path', '')
+                metadata['defaultPassword'] = metadata.get('defaultPassword', '')
+                if metadata['defaultPassword'] == "$APP_SEED":
+                    metadata['defaultPassword'] = deriveEntropy("app-{}-seed".format(app))
+                if "mainContainer" in metadata:
+                    metadata.pop("mainContainer")
+                app_metadata.append(metadata)
+                if(app_yml["version"] != 3):
+                    getPortsOldApp(app_yml, app)
+                else:
+                    getPortsV3App(app_yml, app)
+            except Exception as e:
+                print(e)
+                print("App {} is invalid!".format(app))
+    appPortsToMap()
+    return {
+        "metadata": app_metadata,
+        "ports": appPortMap
+    }
+
+citadelPorts = [
+    # Dashboard
+    80,
+    # Sometimes used by nginx with some setups
+    433,
+    # Dashboard SSL
+    443,
+    # Bitcoin Core P2P
+    8333,
+    # LND gRPC
+    10009,
+    # LND REST
+    8080,
+    # Electrum Server
+    50001,
+    # Tor Proxy
+    9050,
+]
+
+lastPort = 3000
+
+def getNewPort(usedPorts):
+    lastPort2 = lastPort
+    while lastPort2 in usedPorts or lastPort2 in citadelPorts:
+        lastPort2 = lastPort2 + 1
+    return lastPort2
+
+def validatePort(appContainer, port, appId, priority: int, isDynamic = False): 
+    if port not in appPorts and port not in citadelPorts and port != 0:
+        appPorts[port] = {
+            "app": appId,
+            "port": port,
+            "container": appContainer["name"],
+            "priority": priority,
+            "dynamic": isDynamic,
+        }
+    else:
+        if port in citadelPorts or (appPorts[port]["app"] != appId and appPorts[port]["container"] != appContainer["name"]):
+            newPort = getNewPort(appPorts.keys())
+            if port in appPorts and priority > appPorts[port]["priority"]:
+                #print("Prioritizing app {} over {}".format(appId, appPorts[port]["app"]))
+                appPorts[newPort] = appPorts[port].copy()
+                appPorts[port]  = {
+                    "app": appId,
+                    "port": port,
+                    "container": appContainer["name"],
+                    "priority": priority,
+                    "dynamic": isDynamic,
+                }
+            else:
+                if "requiresPort" in appContainer and appContainer["requiresPort"]:
+                    disabledApps.append(appId)
+                    print("App {} disabled because of port conflict".format(appId))
+                else:
+                    #print("Port conflict! Moving app {}'s container {} to port {} (from {})".format(appId, appContainer["name"], newPort, port))
+                    appPorts[newPort]  = {
+                        "app": appId,
+                        "port": port,
+                        "container": appContainer["name"],
+                        "priority": priority,
+                        "dynamic": isDynamic,
+                    }
+
+def getPortsOldApp(app, appId):
+    for appContainer in app["containers"]:
+        if "port" in appContainer:
+            validatePort(appContainer, appContainer["port"], appId, 0)
+        if "ports" in appContainer:
+            for port in appContainer["ports"]:
+                realPort = int(str(port).split(":")[0])
+                validatePort(appContainer, realPort, appId, 2)
+
+
+def getPortsV3App(app, appId):
+    for appContainer in app["containers"]:
+        if "port" in appContainer:
+            if "preferredOutsidePort" in appContainer and "requirsesPort" in appContainer and appContainer["requiresPort"]:
+                validatePort(appContainer, appContainer["preferredOutsidePort"], appId, 2)
+            elif "preferredOutsidePort" in appContainer:
+            
+                validatePort(appContainer, appContainer["preferredOutsidePort"], appId, 1)
+            else:
+                validatePort(appContainer, appContainer["port"], appId, 0)
+        elif "requiredPorts" not in appContainer:
+                validatePort(appContainer, getNewPort(appPorts.keys()), appId, 0, True)
+        if "requiredPorts" in appContainer:
+            for port in appContainer["requiredPorts"]:
+                realPort = int(str(port).split(":")[0])
+                validatePort(appContainer, realPort, appId, 2)
--- a/app/lib/validate.py
+++ b/app/lib/validate.py
@ -16,6 +16,8 @@ def validateApp(app: dict):
        schemaVersion1 = yaml.safe_load(f)
    with open(os.path.join(scriptDir, 'app-standard-v2.yml'), 'r') as f:
        schemaVersion2 = yaml.safe_load(f)
+    with open(os.path.join(scriptDir, 'app-standard-v3.yml'), 'r') as f:
+        schemaVersion3 = yaml.safe_load(f)

    if 'version' in app and str(app['version']) == "1":
        try:
@ -33,6 +35,14 @@ def validateApp(app: dict):
        except Exception as e:
            print(e)
            return False
+    elif 'version' in app and str(app['version']) == "3":
+        try:
+            validate(app, schemaVersion3)
+            return True
+        # Catch and log any errors, and return false
+        except Exception as e:
+            print(e)
+            return False
    else:
        print("Unsupported app version")
        return False