citadel-core/scripts/debug

#!/usr/bin/env bash

# SPDX-FileCopyrightText: 2021-2022 Citadel and contributors
#
# SPDX-License-Identifier: GPL-3.0-or-later

NODE_ROOT="$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/.."
cd "${NODE_ROOT}"

if [[ $UID != 0 ]]; then
    echo "This script must run as root"
    echo "Please re-run this script as"
    echo "  sudo ~/citadel/scripts/debug"
    exit 1
fi

# Try to load Citadel OS version on Citadel OS
[[ -f "/etc/default/citadel" ]] && source "/etc/default/citadel"
[[ -f "./.env" ]] && source "./.env"

# function to upload the output to our debug log server
# Based on https://github.com/seejohnrun/haste-client#lightweight-alternative
upload() {
    echo "Generating link..."
    curl \
    --header "Content-Type: text/plain" \
    --request POST \
    --data-binary @- \
    https://citadel-debug-api.vercel.app/api/upload \
    | jq '.key' \
    | sed 's/"//g' \
    | awk -F '"' '{print "https://logs.runcitadel.space/"$1}'
}

torupload() {
    echo "Generating link..."
    curl \
    --header "Content-Type: text/plain" \
    --request POST \
    --socks5 "localhost:${TOR_PROXY_PORT}" \
    --data-binary @- \
    https://citadel-debug-api.vercel.app/api/upload \
    | jq '.key' \
    | sed 's/"//g' \
    | awk -F '"' '{print "https://logs.runcitadel.space/"$1}'
}

list_block_devices () {
    # Documented in the mount script for the OS
    sync
    (ls -d /sys/block/sd* /sys/block/nvme*n* 2>/dev/null || true) | sed 's!.*/!!'
}

if [[ "${1}" == "--run" ]]; then
    echo "=============="
    echo "= Debug info ="
    echo "=============="
    
    echo
    echo "Node version"
    echo "------------"
    cat info.json | jq .version | sed 's/"//g'
    
    if [[ ! -z "${CITADEL_OS:-}" ]]; then
        echo
        echo "Flashed OS version"
        echo "-----------------"
        echo $CITADEL_OS
        
        echo
        echo "Raspberry Pi Model"
        echo "------------------"
        
        cat /proc/cpuinfo | tail -1
        
        echo
        echo "Temperature"
        echo "-----------"
        
        vcgencmd measure_temp
        
        echo
        echo "Throttling"
        echo "----------"
        
        vcgencmd get_throttled
    fi
    
    echo
    echo "Memory usage"
    echo "------------"
    free --human --mega
    echo
    "${NODE_ROOT}/scripts/memory-usage"
    
    echo
    echo "Memory monitor logs"
    echo "-------------------"
    tail -n 10 "${NODE_ROOT}/logs/memory-monitor.log"
    
    echo
    echo "Filesystem information"
    echo "----------------------"
    
    df --human-readable / "${NODE_ROOT}"
    
    if [[ ! -z "${CITADEL_OS:-}" ]]; then
        echo
        echo "Startup service logs"
        echo "--------------------"
        journalctl --unit=citadel-startup.service | tail -n 30  | sed '/onion/d'
        
        echo
        echo "External storage service logs"
        echo "-----------------------------"
        journalctl --unit=citadel-external-storage.service | tail -n 30
        
        echo
        echo "External storage SD card update service logs"
        echo "--------------------------------------------"
        journalctl --unit=citadel-external-storage-sdcard-update.service | tail -n 30
    fi
    
    
    echo
    echo "Karen logs"
    echo "----------"
    echo
    
    tail -n 50 logs/karen.log | sed '/onion/d'
    
    echo
    echo "Docker containers"
    echo "-----------------"
    
    docker ps --format="table {{.Names}}\t{{.Status}}"
    
    echo
    echo "Bitcoin Core logs"
    echo "-----------------"
    echo
    docker compose logs --tail=30 bitcoin
    
    echo
    echo "Lightning logs"
    echo "--------"
    echo
    docker compose logs --tail=30 lightning
    
    echo
    echo "Tor logs"
    echo "--------"
    echo
    docker compose logs --tail=30 tor
    
    installed_apps=$(./scripts/app ls-installed)
    if [[ ! -z "${installed_apps:-}" ]]; then
        echo
        echo "App logs"
        echo "--------"
        for app in $installed_apps; do
            echo
            echo "${app}"
            echo
            # Double tail because we want 30 lines total not per container
            ./scripts/app compose $app logs | tail -n 30;
        done
    fi
else
    filter_sensitive=false
    allow_colors=true
    upload=false
    instructions=true
    dmesg=false
    tor=true
    while [[ "$#" -gt 0 ]]; do
        case $1 in
            --no-colors) allow_colors=false ;;
            --no-instructions) instructions=false ;;
            -f|--filter) filter_sensitive=true ;;
            -u|--upload) upload=true ;;
            --no-tor) tor=false ;;
            -d|--dmesg) dmesg=true ;;
            -r|--run) echo "--run can't be used with other options!"; exit 1 ;;
            *) echo "Unknown parameter passed: $1"; exit 1 ;;
        esac
        shift
    done
    echo "Loading data..."
    OWN_OUTPUT="$(./scripts/debug --run)"
    append_dmesg () {
        echo "${1}"
        echo "dmesg"
        echo "-----"
        dmesg
    }
    
    if [[ $allow_colors = false ]]; then
        OWN_OUTPUT=$(echo "${OWN_OUTPUT}" | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g")
    fi
    
    if [[ $filter_sensitive = true ]]; then
        # TODO: Add more filters
        OWN_OUTPUT=$(echo "${OWN_OUTPUT}" | sed '/onion/d')
    fi
    
    if [[ $dmesg = true ]]; then
        OWN_OUTPUT=$(append_dmesg "${OWN_OUTPUT}")
    fi
    
    echo "${OWN_OUTPUT}"
    
    if [[ $upload = true ]]; then
        if [[ $dmesg = false ]]; then
            OWN_OUTPUT=$(append_dmesg "${OWN_OUTPUT}")
        fi
        OWN_OUTPUT=$(echo "${OWN_OUTPUT}" | sed '/onion/d')
    fi
    
    echo "================"
    echo "==== Result ===="
    echo "================"
    echo
    if [[ ! -z "${BITCOIN_NODE_OS:-}" ]]; then
        if [[ $(vcgencmd get_throttled) != "throttled=0x0" ]]; then
            if [[ $(vcgencmd get_throttled) = "throttled=0x5"* ]]; then
                echo "There is probably issue with your power supply. We recommend using the official power supply to avoid this issue."
            else
                echo "The temperature of your Raspberry Pi is too high. We recommend either using a case that lowers the temperature of your Pi or a case with a builtin fan."
            fi
            exit 0
        fi
        
        no_of_block_devices=$(list_block_devices | wc -l)
        if [[ $no_of_block_devices -lt 1 ]]; then
            echo "You have either no external drive connected to your Raspberry Pi or the drive is not being detected."
            echo "Please shutdown your Raspberry Pi ('sudo shutdown'), then plug the SSD into the other USB3 port."
            echo "After you've finished that, boot your Raspberry Pi back up. If it still still doesn't work,"
            echo "you can contact us on Telegram."
            if [[ $upload = true ]]; then
                echo "To make it possible for us to analyze your problem easier, please share the following link with us:"
                echo "${OWN_OUTPUT}" | upload
                elif [[ $instructions = true ]]; then
                echo "Please run this script again with the --upload flag to generate a link to share."
            fi
            exit 0
            elif [[ $no_of_block_devices -gt 1 ]]; then
            echo "You have multiple external drives connected to your Raspberry Pi. Please remove the drive(s) you don't want to use for Citadel,"
            echo "then reboot your Raspberry Pi by running 'sudo reboot'."
            exit 0
        fi
    fi
    
    echo "The debug script did not automatically detect any issues with your Citadel."
    if [[ $upload = true ]]; then
        echo "Please copy the following link and share it with us on Telegram so we can help you with your problem."
        if [[ $tor = true ]]; then
            echo "${OWN_OUTPUT}" | torupload
        else
            echo "${OWN_OUTPUT}" | upload
        fi
        elif [[ $instructions = true ]]; then
        echo "Please copy the entire output of this script and share it with us on Telegram so we can help you with your problem."
        echo "It's recommended to upload the output somewhere and share a link to it. Run this script with '--upload' to automatically generate a link to share."
    fi
fi