citadel-core/scripts/configure

#!/usr/bin/env python3

# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later

import sys
import os
from lib.rpcauth import get_data
import re
import subprocess
import json
import shutil
import yaml

# Print an error if the user isn't running on Linux.
if sys.platform != 'linux':
    print('This script only works on Linux!')
    exit(1)

# Print an error if user is not root
if os.getuid() != 0:
  print('This script must be run as root!')
  exit(1)

# Check if the system is arm64 or amd64
is_arm64 = subprocess.check_output(['uname', '-m']).decode('utf-8').strip() == 'aarch64'
is_amd64 = subprocess.check_output(['uname', '-m']).decode('utf-8').strip() == 'x86_64'

if not is_arm64 and not is_amd64:
  print('Citadel only works on arm64 and amd64!')
  exit(1)

# Check the output of "docker compose version", if it matches "Docker Compose version v2.0.0-rc.3", return true
# Otherwise, return false
def is_compose_rc_or_outdated():
  try:
    output = subprocess.check_output(['docker', 'compose', 'version'])
    if output.decode('utf-8').strip() != 'Docker Compose version v2.2.3':
      print("Using outdated docker compose, updating...")
      return True
    else:
      return False
  except:
    return True

# Download docker-compose from GitHub and put it in $HOME/.docker/cli-plugins/docker-compose
def download_docker_compose():
  # Skip if os.path.expanduser('~/.docker/cli-plugins/docker-compose') exists
  subprocess.check_call(["mkdir", "-p", os.path.expanduser('~/.docker/cli-plugins/')])
  if (os.path.exists(os.path.expanduser('~/.docker/cli-plugins/docker-compose')) or os.path.exists('/usr/lib/docker/cli-plugins/docker-compose')) and not is_compose_rc_or_outdated():
    return
  if is_arm64:
    subprocess.check_call(['wget', 'https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-aarch64', '-O', os.path.expanduser('~/.docker/cli-plugins/docker-compose')])
  elif is_amd64:
    subprocess.check_call(['wget', 'https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64', '-O', os.path.expanduser('~/.docker/cli-plugins/docker-compose')])
  os.chmod(os.path.expanduser('~/.docker/cli-plugins/docker-compose'), 0o755)

if not shutil.which("wget"):
    print('Wget is not installed!')
    exit(1)

if not shutil.which("docker"):
    print('Docker is not installed!')
    exit(1)

# Switch to node root directory.
CITADEL_ROOT=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.chdir(CITADEL_ROOT)

updating = False
status_dir = os.path.join(CITADEL_ROOT, 'statuses')
# Make sure to use the main status dir for updates
if os.path.isfile('../.citadel'):
    status_dir = os.path.join(CITADEL_ROOT, '..', 'statuses')
    updating = True

# Configure for mainnet or testnet or regtest depending
# upon the user-supplied value of $NETWORK
# If the network is not specified, then use the mainnet
BITCOIN_NETWORK=os.environ.get('NETWORK') or 'mainnet'

# Check if network neither mainnet nor testnet nor regtest
if BITCOIN_NETWORK not in ['mainnet', 'testnet', 'regtest']:
    print('Error: Network must be either mainnet, testnet, or regtest!')
    exit(1)

with open(os.path.join(CITADEL_ROOT, "info.json"), 'r') as file:
  CITADEL_VERSION=json.load(file)['version']

print()
print("======================================")
if os.path.isfile(status_dir+'/configured'):
  print("=========== RECONFIGURING ============")
  reconfiguring=True
else:
  print("============ CONFIGURING =============")
  reconfiguring=False

print("============== CITADEL  ==============")
print("======================================")
print()

if not reconfiguring:
  print("Installing electrs...")
  data = subprocess.run("\"{}\" install electrs".format(os.path.join(CITADEL_ROOT, "services", "manage.py")), shell=True)

print("Installing additional services")
data = subprocess.run("\"{}\" setup".format(os.path.join(CITADEL_ROOT, "services", "manage.py")), shell=True)

# Parse a dotenv file
# Values can either be KEY=VALUE or KEY="VALUE" or KEY='VALUE'
# Returns all env vars as a dict
def parse_dotenv(file_path):
  envVars: dict = {}
  with open(file_path, 'r') as file:
    for line in file:
      line = line.strip()
      if line.startswith('#') or len(line) == 0:
        continue
      if '=' in line:
        key, value = line.split('=', 1)
        value = value.strip('"').strip("'")
        envVars[key] = value
      else:
        print("Error: Invalid line in {}: {}".format(file_path, line))
        print("Line should be in the format KEY=VALUE or KEY=\"VALUE\" or KEY='VALUE'")
        exit(1)
  return envVars

KNOTS_REINDEX_AUTO="reindex=auto"
BITCOIN_CORE_IMAGE="lncm/bitcoind:v22.0@sha256:37a1adb29b3abc9f972f0d981f45e41e5fca2e22816a023faa9fdc0084aa4507"
if os.path.isfile('../use-core-upstream') or os.path.isfile('./use-core-upstream'):
  KNOTS_REINDEX_AUTO=""
  # Also, open the docker-compose file and replace the image of the bitcoin service 
  # with the upstream version of bitcoin core
  with open(os.path.join(CITADEL_ROOT, "docker-compose.yml"), 'r') as file:
    docker_compose_yml = yaml.safe_load(file)
  docker_compose_yml['services']['bitcoin']['image'] = BITCOIN_CORE_IMAGE
  with open(os.path.join(CITADEL_ROOT, "docker-compose.yml"), 'w') as file:
    yaml.dump(docker_compose_yml, file, sort_keys=False)

##########################################################
############ Generate configuration variables ############
##########################################################

NGINX_PORT=os.environ.get('NGINX_PORT') or "80"
UPDATE_CHANNEL="main"

if reconfiguring:
  if os.path.isfile('../.citadel'):
    dotenv=parse_dotenv('../.env')
  else:
    dotenv=parse_dotenv('./.env')

  BITCOIN_NETWORK=os.environ.get('OVERWRITE_NETWORK') or dotenv['BITCOIN_NETWORK']
  # Check if network neither mainnet nor testnet nor regtest
  if BITCOIN_NETWORK not in ['mainnet', 'testnet', 'regtest']:
      print('Error: Network must be either mainnet, testnet, or regtest!')
      exit(1)
  print("Using {} network".format(BITCOIN_NETWORK))
  print()
  BITCOIN_RPC_PORT=dotenv['BITCOIN_RPC_PORT']
  BITCOIN_P2P_PORT=dotenv['BITCOIN_P2P_PORT']
  BITCOIN_RPC_USER=dotenv['BITCOIN_RPC_USER']
  BITCOIN_RPC_PASS=dotenv['BITCOIN_RPC_PASS']
  BITCOIN_RPC_AUTH=dotenv['BITCOIN_RPC_AUTH']
  TOR_PASSWORD=dotenv['TOR_PASSWORD']
  TOR_HASHED_PASSWORD=dotenv['TOR_HASHED_PASSWORD']
  NGINX_PORT=dotenv['NGINX_PORT']
  if 'UPDATE_CHANNEL' in dotenv:
    UPDATE_CHANNEL=dotenv['UPDATE_CHANNEL']
else:
  # Generate RPC credentials
  print("Generating auth credentials")
  print()
  BITCOIN_RPC_USER="citadel"
  BITCOIN_RPC_DETAILS=get_data(BITCOIN_RPC_USER)
  BITCOIN_RPC_AUTH=BITCOIN_RPC_DETAILS['auth']
  BITCOIN_RPC_PASS=BITCOIN_RPC_DETAILS['password']

  # Pull Tor image and generate Tor password
  print("Generating Tor password")
  print()
  os.system('docker pull --quiet lncm/tor:0.4.6.8')
  TOR_PASSWORD=get_data('itdoesntmatter')['password']

  # run 'docker run --rm lncm/tor:0.4.6.7 --quiet --hash-password "$TOR_PASS"' and get its output
  # this is the password that is used to connect to the Tor network
  # the password is stored in the .env file
  TOR_HASHED_PASSWORD=os.popen('docker run --rm lncm/tor:0.4.6.8 --quiet --hash-password "{}"'.format(TOR_PASSWORD)).read()[:-1]

BITCOIN_NODE="neutrino"
ALIAS_AND_COLOR=""
ADDITIONAL_BITCOIN_OPTIONS=""
BOLT_DB_OPTIONS=""
CHANNEL_LIMITATIONS=""
BASEFEE = "bitcoin.basefee=0"
EXTERNAL_IP = ""
if os.path.isfile('./tor/data/bitcoin-p2p/hostname'):
  EXTERNAL_IP="externalip=" + open('./tor/data/bitcoin-p2p/hostname').read()

if os.path.isfile("./lnd/lnd.conf"):
  with open("./lnd/lnd.conf", 'r') as file:
    # We generally don't want to allow changing lnd.conf, but we keep as many custom settings as possible
    for line in file:
      if line.startswith("bitcoin.node="):
        BITCOIN_NODE = line.split("=")[1].strip()
      if line.startswith("alias="):
        ALIAS_AND_COLOR += "\n" + line.strip()
      if line.startswith("color="):
        ALIAS_AND_COLOR += "\n" + line.strip()
      if line.startswith("bitcoin.basefee"):
        BASEFEE = line.strip()
      if line.startswith("bitcoin.feerate"):
        ADDITIONAL_BITCOIN_OPTIONS += "\n" + line.strip()
      if line.startswith("minchansize"):
        CHANNEL_LIMITATIONS += "\n" + line.strip()
      if line.startswith("maxchansize"):
        CHANNEL_LIMITATIONS += "\n" + line.strip()
      if line.startswith("maxpendingchannels"):
        CHANNEL_LIMITATIONS += "\n" + line.strip()
      if line.startswith("db.bolt.auto-compact"):
        BOLT_DB_OPTIONS += "\n" + line.strip()

if BOLT_DB_OPTIONS != "":
  BOLT_DB_OPTIONS = "[bolt]\n" + BOLT_DB_OPTIONS

if CHANNEL_LIMITATIONS == "":
  CHANNEL_LIMITATIONS = "maxpendingchannels=3\nminchansize=10000"

NEUTRINO_PEERS=""
if BITCOIN_NETWORK == "mainnet":
  BITCOIN_RPC_PORT=8332
  BITCOIN_P2P_PORT=8333
elif BITCOIN_NETWORK == "testnet":
  BITCOIN_RPC_PORT=18332
  BITCOIN_P2P_PORT=18333
  NEUTRINO_PEERS='''
[neutrino]
neutrino.addpeer=testnet1-btcd.zaphq.io
neutrino.addpeer=testnet2-btcd.zaphq.io
  '''
elif BITCOIN_NETWORK == "regtest":
  BITCOIN_RPC_PORT=18334
  BITCOIN_P2P_PORT=18335
  BITCOIN_NODE="bitcoind"
else:
  exit(1)

NETWORK_SECTION=""
if BITCOIN_NETWORK != "mainnet":
  NETWORK_SECTION = "[{}]".format(BITCOIN_NETWORK)

# IP addresses for services
NETWORK_IP="10.21.21.0"
GATEWAY_IP="10.21.21.1"
NGINX_IP="10.21.21.2"
DASHBOARD_IP="10.21.21.3"
MANAGER_IP="10.21.21.4"
MIDDLEWARE_IP="10.21.21.5"
NEUTRINO_SWITCHER_IP="10.21.21.6"
BITCOIN_IP="10.21.21.8"
LND_IP="10.21.21.9"
ELECTRUM_IP="10.21.21.10"
TOR_PROXY_IP="10.21.21.11"
APPS_TOR_IP="10.21.21.12"
APPS_2_TOR_IP="10.21.21.13"
APPS_3_TOR_IP="10.21.21.14"
REDIS_IP="10.21.21.15"

# Ports
BITCOIN_RPC_PORT="8332"
BITCOIN_P2P_PORT="8333"
BITCOIN_ZMQ_RAWBLOCK_PORT="28332"
BITCOIN_ZMQ_RAWTX_PORT="28333"
BITCOIN_ZMQ_HASHBLOCK_PORT="28334"
LND_GRPC_PORT="10009"
LND_REST_PORT="8080"
ELECTRUM_PORT="50001"
TOR_PROXY_PORT="9050"
TOR_CONTROL_PORT="29051"

DEVICE_HOSTNAME=""
try:
  subprocess.check_output("hostname").decode("utf-8").strip()
except:
  # The content of /etc/hostname is the device's hostname
  DEVICE_HOSTNAME=open("/etc/hostname").read().strip()

DOCKER_EXECUTABLE=subprocess.check_output(["which", "docker"]).decode("utf-8").strip()
# Get the real path by following symlinks
DOCKER_BINARY=subprocess.check_output(["readlink",  "-f", DOCKER_EXECUTABLE]).decode("utf-8").strip()

# Set LND fee URL for neutrino on mainnet
LND_FEE_URL=""
# If the network is mainnet and status_dir/node-status-bitcoind-ready doesn't exist, set the FEE_URL
if BITCOIN_NETWORK == 'mainnet' and BITCOIN_NODE == 'neutrino':
   LND_FEE_URL="feeurl=https://nodes.lightning.computer/fees/v1/btc-fee-estimates.json"

# Checks if a variable with the name exists, if not, check if an env var with the name existts
# if neither exists, then exit with an error
def get_var(var_name, other_locals, file_name):
  if var_name in locals():
    return str(locals()[var_name])
  elif var_name in other_locals:
    return str(other_locals[var_name])
  elif var_name in globals():
    return str(globals()[var_name])
  else:
    print("Error: {} is not defined! (In file {})".format(var_name, file_name))
    exit(1)

# Converts a string to uppercase, also replaces all - with _
def convert_to_upper(string):
  return string.upper().replace('-', '_')

# Put variables in the config file. A config file accesses an env var $EXAMPLE_VARIABLE by containing <example-variable>
# in the config file. Check for such occurences and replace them with the actual variable
def replace_vars(file_path):
  with open(file_path, 'r') as file:
    file_contents = file.read()
  return re.sub(r'<(.*?)>', lambda m: get_var(convert_to_upper(m.group(1)), locals(), file_path), file_contents)

templates_to_build = {
  "./templates/torrc-empty": ["./tor/torrc-apps", "./tor/torrc-apps-2", "./tor/torrc-apps-3"],
  "./templates/torrc-core-sample": "./tor/torrc-core",
  "./templates/lnd-sample.conf": "./lnd/lnd.conf",
  "./templates/bitcoin-sample.conf": "./bitcoin/bitcoin.conf",
  "./templates/.env-sample": "./.env",
  "./templates/electrs-sample.toml": "./electrs/electrs.toml",
  "./templates/nginx-sample.conf": "./nginx/nginx.conf"
}

print("Generating configuration files...")
print()

# Loop through templates_to_build and build them
for template_path, output_path in templates_to_build.items():
  if output_path == "./nginx/nginx.conf" and os.path.isfile(output_path):
    continue
  data = replace_vars(template_path)
  # If output path is a list, then it is a list of output paths
  if isinstance(output_path, list):
    for output_path_item in output_path:
      # Delete the output path, no matter if it's a file or a directory
      if os.path.isdir(output_path_item):
        shutil.rmtree(output_path_item)
      with open(output_path_item, 'w') as file:
        file.write(data)
  else:
    # Delete the output path, no matter if it's a file or a directory
    if os.path.isdir(output_path):
      shutil.rmtree(output_path)
    with open(output_path, 'w') as file:
      file.write(data)

print("Generated configuration files")
print()

print("Installing docker-compose...")
print()
download_docker_compose()
if not reconfiguring:
  print("Updating apps...")
  print()
  os.system('./scripts/app --invoked-by-configure update')
elif not updating:
  print("Updating apps...")
  print()
  os.system('./scripts/app --invoked-by-configure generate')
print("Configuring permissions")
print()
os.system('chown -R 1000:1000 {}'.format(CITADEL_ROOT))
# Touch status_dir/configured
with open(status_dir+'/configured', 'w') as file:
  file.write('')
print("Configuration successful")
print("You can now start Citadel by running:")
print("  sudo ./scripts/start")