From e77fb024e66ff7cda2fab0d93b64375576db458a Mon Sep 17 00:00:00 2001 From: Taylor Helsper Date: Sun, 6 Mar 2022 22:02:30 -0600 Subject: [PATCH] More Premium+; Python refactor --- rootfs/standard/usr/bin/mynode_backup_scb.py | 40 +++- .../standard/usr/bin/mynode_post_upgrade.sh | 2 +- .../usr/bin/mynode_premium_plus_connect.py | 52 ++++- .../btcpayserver/docker-compose.generated.yml | 12 +- rootfs/standard/usr/share/btcpayserver/env | 2 +- .../usr/share/mynode/application_info.json | 1 - rootfs/standard/usr/share/mynode/lnd.conf | 5 +- .../usr/share/mynode/lnd_watchtower.conf | 2 - .../mynode => pynode}/application_info.py | 40 +++- rootfs/standard/var/pynode/bitcoin_info.py | 84 +++++++-- rootfs/standard/var/pynode/device_info.py | 49 +++++ rootfs/standard/var/pynode/drive_info.py | 23 ++- rootfs/standard/var/pynode/lightning_info.py | 22 +++ rootfs/standard/var/pynode/utilities.py | 93 ++++++++- rootfs/standard/var/www/mynode/api.py | 1 + rootfs/standard/var/www/mynode/bitcoin.py | 26 +-- rootfs/standard/var/www/mynode/mynode.py | 4 + .../standard/var/www/mynode/premium_plus.py | 57 +++++- .../var/www/mynode/templates/bitcoin.html | 2 +- .../var/www/mynode/templates/lnd.html | 2 +- .../www/mynode/templates/premium_plus.html | 177 ++++++++++++++++-- .../var/www/mynode/thread_functions.py | 44 ----- 22 files changed, 605 insertions(+), 135 deletions(-) rename rootfs/standard/var/{www/mynode => pynode}/application_info.py (90%) diff --git a/rootfs/standard/usr/bin/mynode_backup_scb.py b/rootfs/standard/usr/bin/mynode_backup_scb.py index ceab1b32..fc129755 100755 --- a/rootfs/standard/usr/bin/mynode_backup_scb.py +++ b/rootfs/standard/usr/bin/mynode_backup_scb.py @@ -26,7 +26,13 @@ log.setLevel(logging.INFO) set_logger(log) # Helper functions - +cached_remote_hash = "NONE" +def get_saved_remote_backup_hash(): + global cached_remote_hash + return cached_remote_hash +def set_saved_remote_backup_hash(hash): + global cached_remote_hash + cached_remote_hash = hash # Local Backup def local_backup(original_scb, backup_scb): @@ -51,21 +57,40 @@ def local_backup(original_scb, backup_scb): # Remote Backup def remote_backup(original, backup): + # Check if remote backup is enabled + premium_plus_settings = get_premium_plus_settings() + if not premium_plus_settings['backup_scb']: + log_message("Remote Backup: SCB Backup Disabled") + return + # Mainnet only if is_testnet_enabled(): - log_message("Remote Backup: Skipping (testnet enabled") + log_message("Remote Backup: Skipping (testnet enabled)") return # Premium+ Feature - if not has_premium_plus_token() or get_premium_plus_token_status != "OK": + if not has_premium_plus_token() or get_premium_plus_token_status() != "OK": log_message("Remote Backup: Skipping (not Premium+)") return + md5_1 = get_md5_file_hash(original_scb) + md5_2 = get_saved_remote_backup_hash() + log_message(" Hash 1: {}".format(md5_1)) + log_message(" Hash 2: {}".format(md5_2)) + if md5_1 == md5_2: + log_message("Remote Backup: Hashes Match. Skipping Backup.") + return + # POST Data + try: + file_data = {'scb': open(original,'rb')} + except Exception as e: + log_message("Remote Backup: Error reading SCB file.") + return + data = { "token": get_premium_plus_token(), - "product_key": get_product_key(), - "scb_file": "REPLACE ME WITH FILE CONTENTS" + "product_key": get_product_key() } # Setup tor proxy @@ -82,13 +107,14 @@ def remote_backup(original, backup): # Use tor for check in unless there have been tor 5 failures in a row r = None if (fail_count+1) % 5 == 0: - r = requests.post(BACKUP_SCB_URL, data=data, timeout=20) + r = requests.post(BACKUP_SCB_URL, data=data, files=file_data, timeout=20) else: - r = session.post(BACKUP_SCB_URL, data=data, timeout=20) + r = session.post(BACKUP_SCB_URL, data=data, files=file_data, timeout=20) if r.status_code == 200: if r.text == "OK": log_message("Remote Backup: Success ({})".format(r.text)) + set_saved_remote_backup_hash( md5_1 ) else: log_message("Remote Backup: Error: ({})".format(r.text)) backup_success = True diff --git a/rootfs/standard/usr/bin/mynode_post_upgrade.sh b/rootfs/standard/usr/bin/mynode_post_upgrade.sh index fee8f4e6..155927db 100755 --- a/rootfs/standard/usr/bin/mynode_post_upgrade.sh +++ b/rootfs/standard/usr/bin/mynode_post_upgrade.sh @@ -208,7 +208,7 @@ if ! skip_base_upgrades ; then # Remove old python files so new copies are used (files migrated to pynode) set +x PYNODE_FILES="/var/pynode/*.py" - for pynode_file in $PYNODE_FILES + for pynode_file in $PYNODE_FILES; do echo "Migrating pynode file $pynode_file..." pynode_file="$(basename -- $pynode_file)" rm -f /var/www/mynode/${pynode_file} # .py diff --git a/rootfs/standard/usr/bin/mynode_premium_plus_connect.py b/rootfs/standard/usr/bin/mynode_premium_plus_connect.py index 50f112ea..5b1d9d2a 100755 --- a/rootfs/standard/usr/bin/mynode_premium_plus_connect.py +++ b/rootfs/standard/usr/bin/mynode_premium_plus_connect.py @@ -9,6 +9,9 @@ from systemd import journal from utilities import * from device_info import * from drive_info import * +from application_info import * +from bitcoin_info import * +from lightning_info import * PREMIUM_PLUS_CONNECT_URL = "https://www.mynodebtc.com/device_api/premium_plus_connect.php" @@ -19,7 +22,46 @@ log.setLevel(logging.INFO) set_logger(log) # Helper functions +def get_premium_plus_device_info(): + info = {} + settings = get_premium_plus_settings() + # Basic Info + info["serial"] = get_device_serial() + info["device_type"] = get_device_type() + info["device_arch"] = get_device_arch() + info["drive_size"] = get_mynode_drive_size() + info["data_drive_usage"] = get_data_drive_usage() + info["os_drive_usage"] = get_os_drive_usage() + info["temperature"] = get_device_temp() + info["total_ram"] = get_device_ram() + info["ram_usage"] = get_ram_usage() + info["swap_usage"] = get_swap_usage() + info["uptime"] = get_system_uptime() + + # App status info + if settings['sync_status']: + info["app_info"] = get_all_applications_from_json_cache() + + return info + +def get_premium_plus_bitcoin_info(): + info = {} + settings = get_premium_plus_settings() + + if settings['sync_bitcoin_and_lightning']: + log_message("BITCOIN " + str(get_bitcoin_json_cache())) + info = get_bitcoin_json_cache() + return info + +def get_premium_plus_lightning_info(): + info = {} + settings = get_premium_plus_settings() + + if settings['sync_bitcoin_and_lightning']: + log_message("LIGHTNING " + str(get_lightning_json_cache())) + info = get_lightning_json_cache() + return info # Update hourly def premium_plus_connect(): @@ -27,13 +69,13 @@ def premium_plus_connect(): # Check in data = { "serial": get_device_serial(), - "device_type": get_device_type(), - "device_arch": get_device_arch(), "version": get_current_version(), "token": get_premium_plus_token(), + "settings": json.dumps(get_premium_plus_settings()), + "device_info": json.dumps(get_premium_plus_device_info()), + "bitcoin_info": json.dumps(get_premium_plus_bitcoin_info()), + "lightning_info": json.dumps(get_premium_plus_lightning_info()), "product_key": get_product_key(), - "drive_size": get_mynode_drive_size(), - "drive_usage": get_mynode_drive_usage(), } # Setup tor proxy @@ -105,7 +147,7 @@ if __name__ == "__main__": wd = inotify.add_watch('/home/bitcoin/.mynode/', watch_flags) for event in inotify.read(timeout=one_hour_in_ms): log_message("File changed: " + str(event)) - log_message("Running connect again: " + str(event)) + log_message("Running connect again") except Exception as e: log_message("Error: {}".format(e)) time.sleep(60) diff --git a/rootfs/standard/usr/share/btcpayserver/docker-compose.generated.yml b/rootfs/standard/usr/share/btcpayserver/docker-compose.generated.yml index 9e7caed5..3b136a5c 100644 --- a/rootfs/standard/usr/share/btcpayserver/docker-compose.generated.yml +++ b/rootfs/standard/usr/share/btcpayserver/docker-compose.generated.yml @@ -16,6 +16,7 @@ services: BTCPAY_SSHKEYFILE: ${BTCPAY_SSHKEYFILE} BTCPAY_SSHAUTHORIZEDKEYS: ${BTCPAY_SSHAUTHORIZEDKEYS} BTCPAY_DEBUGLOG: btcpay.log + BTCPAY_DOCKERDEPLOYMENT: "true" BTCPAY_CHAINS: "btc" BTCPAY_BTCEXPLORERURL: http://localhost:32838/ BTCPAY_BTCLIGHTNING: "type=lnd-rest;server=https://localhost:10080/;macaroonfilepath=/etc/lnd_bitcoin/data/chain/bitcoin/mainnet/admin.macaroon;allowinsecure=true" @@ -36,6 +37,7 @@ services: - "/mnt/hdd/mynode/lnd:/etc/lnd_bitcoin" - "/root/.ssh/authorized_keys:${BTCPAY_SSHAUTHORIZEDKEYS}" - "/root/.ssh/id_rsa_btcpay:${BTCPAY_SSHKEYFILE}" + - "btcpay_pluginsdir:/root/.btcpayserver/Plugins" nbxplorer: restart: unless-stopped image: nicolasdorier/nbxplorer:${NBXPLORER_VERSION} @@ -45,17 +47,23 @@ services: environment: NBXPLORER_NETWORK: ${NBITCOIN_NETWORK:-regtest} NBXPLORER_BIND: 0.0.0.0:32838 + NBXPLORER_TRIMEVENTS: 10000 NBXPLORER_SIGNALFILESDIR: /datadir NBXPLORER_CHAINS: "btc" NBXPLORER_BTCRPCURL: http://localhost:8332/ NBXPLORER_BTCNODEENDPOINT: localhost:8333 volumes: - "nbxplorer_datadir:/datadir" - - "/mnt/hdd/mynode/bitcoin:/root/.bitcoin" + + + + postgres: restart: unless-stopped - image: postgres:${POSTGRES_VERSION} + image: btcpayserver/postgres:${POSTGRES_VERSION} network_mode: host + environment: + POSTGRES_HOST_AUTH_METHOD: trust volumes: - "postgres_datadir:/var/lib/postgresql/data" volumes: diff --git a/rootfs/standard/usr/share/btcpayserver/env b/rootfs/standard/usr/share/btcpayserver/env index 2ad40e70..4c382c8d 100644 --- a/rootfs/standard/usr/share/btcpayserver/env +++ b/rootfs/standard/usr/share/btcpayserver/env @@ -24,4 +24,4 @@ TOR_RELAY_EMAIL= # myNode Additions BTCPAY_VERSION=REPLACE_BTCPAY_VERSION NBXPLORER_VERSION=REPLACE_NBXPLORER_VERSION -POSTGRES_VERSION=9.6.5 \ No newline at end of file +POSTGRES_VERSION=13.4.4 \ No newline at end of file diff --git a/rootfs/standard/usr/share/mynode/application_info.json b/rootfs/standard/usr/share/mynode/application_info.json index 69e4d044..57722335 100644 --- a/rootfs/standard/usr/share/mynode/application_info.json +++ b/rootfs/standard/usr/share/mynode/application_info.json @@ -288,7 +288,6 @@ "comment": "HIDDEN FOR NOW - Enable show_on_homepage", "name": "Premium+", "short_name": "premium_plus", - "hide_status_icon": true, "can_enable_disable": false, "app_tile_button_text": "Manage", "app_tile_default_status_text": "Access and Backup", diff --git a/rootfs/standard/usr/share/mynode/lnd.conf b/rootfs/standard/usr/share/mynode/lnd.conf index 672007d3..9e51ebb5 100644 --- a/rootfs/standard/usr/share/mynode/lnd.conf +++ b/rootfs/standard/usr/share/mynode/lnd.conf @@ -50,4 +50,7 @@ bitcoind.rpchost=127.0.0.1:8332 db.bolt.auto-compact=true [routing] -routing.strictgraphpruning=true \ No newline at end of file +routing.strictgraphpruning=true + +[Wtclient] +wtclient.active=1 \ No newline at end of file diff --git a/rootfs/standard/usr/share/mynode/lnd_watchtower.conf b/rootfs/standard/usr/share/mynode/lnd_watchtower.conf index 53cea1cb..a3310530 100644 --- a/rootfs/standard/usr/share/mynode/lnd_watchtower.conf +++ b/rootfs/standard/usr/share/mynode/lnd_watchtower.conf @@ -2,5 +2,3 @@ [Watchtower] watchtower.active=1 -[Wtclient] -wtclient.active=1 diff --git a/rootfs/standard/var/www/mynode/application_info.py b/rootfs/standard/var/pynode/application_info.py similarity index 90% rename from rootfs/standard/var/www/mynode/application_info.py rename to rootfs/standard/var/pynode/application_info.py index 8e09aa04..93216430 100644 --- a/rootfs/standard/var/www/mynode/application_info.py +++ b/rootfs/standard/var/pynode/application_info.py @@ -12,6 +12,7 @@ import re import os # Cached data +JSON_APPLICATION_CACHE_FILE = "/tmp/app_cache.json" mynode_applications = None # Utility functions @@ -118,8 +119,8 @@ def initialize_application_defaults(app): if not "show_on_application_page" in app: app["show_on_application_page"] = True if not "can_enable_disable" in app: app["can_enable_disable"] = True if not "is_enabled" in app: app["is_enabled"] = is_service_enabled( app["short_name"] ) - #app["status"] = status # Should status be optional to include? Takes lots of time. - #app["status_color"] = get_service_status_color(short_name) + #app["status"] = get_application_status( app["short_name"] ) + #app["status_color"] = get_service_status_color( app["short_name"] ) if not "hide_status_icon" in app: app["hide_status_icon"] = False if not "log_file" in app: app["log_file"] = get_application_log_file( app["short_name"] ) if not "journalctl_log_name" in app: app["journalctl_log_name"] = None @@ -134,11 +135,12 @@ def initialize_application_defaults(app): return app -def update_application(app): +def update_application(app, include_status=False): short_name = app["short_name"] app["is_enabled"] = is_service_enabled(short_name) - #app["status"] = "???" # Should status be optional to include? Takes lots of time. - #app["status_color"] = get_service_status_color(short_name) + if include_status: + app["status"] = get_application_status( app["short_name"] ) + app["status_color"] = get_service_status_color( app["short_name"] ) def initialize_applications(): global mynode_applications @@ -157,11 +159,11 @@ def initialize_applications(): mynode_applications = copy.deepcopy(apps) return -def update_applications(): +def update_applications(include_status=False): global mynode_applications for app in mynode_applications: - update_application(app) + update_application(app, include_status) def clear_application_cache(): global mynode_applications @@ -180,7 +182,7 @@ def need_application_refresh(): return True return False -def get_all_applications(order_by="none"): +def get_all_applications(order_by="none", include_status=False): global mynode_applications if need_application_refresh(): @@ -188,6 +190,9 @@ def get_all_applications(order_by="none"): else: update_applications() + if include_status: + update_applications(include_status) + apps = copy.deepcopy(mynode_applications) if order_by == "alphabetic": apps.sort(key=lambda x: x["name"]) @@ -196,6 +201,17 @@ def get_all_applications(order_by="none"): return apps +# Only call this from the www python process so status data is available +def update_application_json_cache(): + global JSON_APPLICATION_CACHE_FILE + apps = get_all_applications(order_by="alphabetic", include_status=True) + return set_dictionary_file_cache(apps, JSON_APPLICATION_CACHE_FILE) + +# Getting the data can be called from any process +def get_all_applications_from_json_cache(): + global JSON_APPLICATION_CACHE_FILE + return get_dictionary_file_cache(JSON_APPLICATION_CACHE_FILE) + def get_application(short_name): apps = get_all_applications() for app in apps: @@ -310,6 +326,14 @@ def get_application_status_color_special(short_name): dojo_initialized = "" if dojo_initialized != "true": return "red" + elif short_name == "premium_plus": + if has_premium_plus_token(): + if get_premium_plus_is_connected(): + return "green" + else: + return "red" + else: + return "gray" return "" def get_application_status_color(short_name): diff --git a/rootfs/standard/var/pynode/bitcoin_info.py b/rootfs/standard/var/pynode/bitcoin_info.py index cbb3a9b3..cac93554 100644 --- a/rootfs/standard/var/pynode/bitcoin_info.py +++ b/rootfs/standard/var/pynode/bitcoin_info.py @@ -22,6 +22,7 @@ bitcoin_wallets = None bitcoin_mempool = None bitcoin_recommended_fees = None bitcoin_version = None +BITCOIN_CACHE_FILE = "/tmp/bitcoin_info.json" # Functions def get_bitcoin_rpc_username(): @@ -57,15 +58,24 @@ def update_bitcoin_main_info(): rpc_connection = AuthServiceProxy("http://%s:%s@127.0.0.1:8332"%(rpc_user, rpc_pass), timeout=120) # Basic Info - bitcoin_blockchain_info = rpc_connection.getblockchaininfo() - if bitcoin_blockchain_info != None: - bitcoin_block_height = bitcoin_blockchain_info['headers'] - mynode_block_height = bitcoin_blockchain_info['blocks'] + info = rpc_connection.getblockchaininfo() + if info != None: + # Save specific data + bitcoin_block_height = info['headers'] + mynode_block_height = info['blocks'] + # Data cleanup + if "difficulty" in info: + info["difficulty"] = "{:.3g}".format(info["difficulty"]) + if "verificationprogress" in info: + info["verificationprogress"] = "{:.3g}".format(info["verificationprogress"]) + + bitcoin_blockchain_info = info except Exception as e: log_message("ERROR: In update_bitcoin_info - {}".format( str(e) )) return False + update_bitcoin_json_cache() return True def update_bitcoin_other_info(): @@ -99,14 +109,37 @@ def update_bitcoin_other_info(): bitcoin_recent_blocks = rpc_connection.batch_([ [ "getblock", h ] for h in block_hashes ]) bitcoin_recent_blocks_last_cache_height = mynode_block_height - # Get peers - bitcoin_peers = rpc_connection.getpeerinfo() + # Get peers and cleanup data + peerdata = rpc_connection.getpeerinfo() + peers = [] + if peerdata != None: + for p in peerdata: + peer = p + + peer["pingtime"] = int(p["pingtime"] * 1000) if ("pingtime" in p) else "N/A" + peer["tx"] = "{:.2f}".format(float(p["bytessent"]) / 1000 / 1000) if ("bytessent" in p) else "N/A" + peer["rx"] = "{:.2f}".format(float(p["bytesrecv"]) / 1000 / 1000) if ("bytesrecv" in p) else "N/A" + peer["minping"] = str(p["minping"]) if ("minping" in p) else "N/A" + peer["minfeefilter"] = str(p["minfeefilter"]) if ("minfeefilter" in p) else "N/A" + peer["pingwait"] = str(p["pingwait"]) if ("pingwait" in p) else "N/A" + + peers.append(peer) + bitcoin_peers = peers # Get network info - bitcoin_network_info = rpc_connection.getnetworkinfo() + network_data = rpc_connection.getnetworkinfo() + if network_data != None: + network_data["relayfee"] = str(network_data["relayfee"]) + network_data["incrementalfee"] = str(network_data["incrementalfee"]) + bitcoin_network_info = network_data # Get mempool - bitcoin_mempool = rpc_connection.getmempoolinfo() + mempool_data = rpc_connection.getmempoolinfo() + if mempool_data != None: + mempool_data["total_fee"] = str(mempool_data["total_fee"]) + mempool_data["mempoolminfee"] = str(mempool_data["mempoolminfee"]) + mempool_data["minrelaytxfee"] = str(mempool_data["minrelaytxfee"]) + bitcoin_mempool = mempool_data # Get wallet info wallets = rpc_connection.listwallets() @@ -145,6 +178,7 @@ def update_bitcoin_other_info(): log_message("ERROR: In update_bitcoin_other_info - {}".format( str(e2) )) return False + update_bitcoin_json_cache() return True def get_bitcoin_status(): @@ -169,7 +203,7 @@ def get_bitcoin_blockchain_info(): def get_bitcoin_difficulty(): info = get_bitcoin_blockchain_info() if "difficulty" in info: - return "{:.3g}".format(info["difficulty"]) + return info["difficulty"] return "???" def get_bitcoin_block_height(): @@ -209,17 +243,27 @@ def get_bitcoin_mempool_info(): mempool["size"] = "???" mempool["bytes"] = "0" if mempooldata != None: + mempool["display_size"] = "unknown" if "size" in mempooldata: mempool["size"] = mempooldata["size"] if "bytes" in mempooldata: mempool["bytes"] = mempooldata["bytes"] + mempool["display_bytes"] = "{:.1} MB".format( float(mempooldata["bytes"])/1000/1000 ) return copy.deepcopy(mempool) def get_bitcoin_mempool_size(): info = get_bitcoin_mempool_info() size = float(info["bytes"]) / 1000 / 1000 - return "{:.3} MB".format(size) + return "{:.1} MB".format(size) + +def get_bitcoin_disk_usage(): + info = get_bitcoin_blockchain_info() + if "size_on_disk" in info: + usage = int(info["size_on_disk"]) / 1000 / 1000 / 1000 + return "{:.0f}".format(usage) + else: + return "UNK" def get_bitcoin_recommended_fees(): global bitcoin_recommended_fees @@ -297,4 +341,22 @@ def is_bip158_enabled(): def enable_bip158(): touch("/mnt/hdd/mynode/settings/.bip158_enabled") def disable_bip158(): - delete_file("/mnt/hdd/mynode/settings/.bip158_enabled") \ No newline at end of file + delete_file("/mnt/hdd/mynode/settings/.bip158_enabled") + + +def update_bitcoin_json_cache(): + global BITCOIN_CACHE_FILE + bitcoin_data = {} + bitcoin_data["current_block_height"] = mynode_block_height + bitcoin_data["blockchain_info"] = get_bitcoin_blockchain_info() + #bitcoin_data["recent_blocks"] = bitcoin_recent_blocks + bitcoin_data["peers"] = get_bitcoin_peers() + bitcoin_data["network_info"] = get_bitcoin_network_info() + bitcoin_data["mempool"] = get_bitcoin_mempool_info() + #bitcoin_data["recommended_fees"] = bitcoin_recommended_fees + bitcoin_data["disk_usage"] = get_bitcoin_disk_usage() + return set_dictionary_file_cache(bitcoin_data, BITCOIN_CACHE_FILE) + +def get_bitcoin_json_cache(): + global BITCOIN_CACHE_FILE + return get_dictionary_file_cache(BITCOIN_CACHE_FILE) \ No newline at end of file diff --git a/rootfs/standard/var/pynode/device_info.py b/rootfs/standard/var/pynode/device_info.py index 13c27116..d4307437 100644 --- a/rootfs/standard/var/pynode/device_info.py +++ b/rootfs/standard/var/pynode/device_info.py @@ -12,6 +12,7 @@ import subprocess import random import string import re +import psutil try: import qrcode @@ -315,6 +316,44 @@ def get_device_ram(): cached_data["ram"] = ram return ram +def get_device_temp(): + if is_cached("device_temp", 60): + return get_cached_data("device_temp") + + device_temp = "..." + try: + results = to_string(subprocess.check_output("cat /sys/class/thermal/thermal_zone0/temp", shell=True)) + temp = int(results) / 1000 + device_temp = "{:.1f}".format(temp) + update_cached_data("device_temp", device_temp) + except: + return device_temp + return device_temp + +def get_ram_usage(): + if is_cached("ram_usage", 120): + return get_cached_data("ram_usage") + + ram_usage = "..." + try: + ram_info = psutil.virtual_memory() + ram_usage = "{:.1f}%".format(ram_info.percent) + except: + return ram_usage + return ram_usage + +def get_swap_usage(): + if is_cached("swap_usage", 120): + return get_cached_data("swap_usage") + + swap_usage = "..." + try: + swap_info = psutil.swap_memory() + swap_usage = "{:.1f}%".format(swap_info.percent) + except: + return swap_usage + return swap_usage + def get_local_ip(): local_ip = "unknown" try: @@ -795,6 +834,16 @@ def recheck_premium_plus_token(): reset_premium_plus_token_status() os.system("systemctl restart premium_plus_connect") +def get_premium_plus_setting_names(): + return ["sync_status","sync_bitcoin_and_lightning","backup_scb","watchtower"] +def get_premium_plus_settings(): + names = get_premium_plus_setting_names() + settings = {} + for n in names: + settings[n] = False + for n in names: + settings[n] = settings_file_exists(n) + return settings #================================== # Drive Repair Functions diff --git a/rootfs/standard/var/pynode/drive_info.py b/rootfs/standard/var/pynode/drive_info.py index b92adc37..09c4d376 100644 --- a/rootfs/standard/var/pynode/drive_info.py +++ b/rootfs/standard/var/pynode/drive_info.py @@ -52,8 +52,27 @@ def get_mynode_drive_size(): size = -2 return size -def get_mynode_drive_usage(): - return "TODO" +def get_data_drive_usage(): + if is_cached("data_drive_usage", 300): + return get_cached_data("data_drive_usage") + usage = "0%" + try: + usage = to_string(subprocess.check_output("df -h /mnt/hdd | grep /dev | awk '{print $5}'", shell=True)) + update_cached_data("data_drive_usage", usage) + except: + return usage + return usage + +def get_os_drive_usage(): + if is_cached("os_drive_usage", 300): + return get_cached_data("os_drive_usage") + usage = "0%" + try: + usage = to_string(subprocess.check_output("df -h / | grep /dev | awk '{print $5}'", shell=True)) + update_cached_data("os_drive_usage", usage) + except: + return usage + return usage def check_partition_for_mynode(partition): is_mynode = False diff --git a/rootfs/standard/var/pynode/lightning_info.py b/rootfs/standard/var/pynode/lightning_info.py index e0a67f76..ea7a586d 100644 --- a/rootfs/standard/var/pynode/lightning_info.py +++ b/rootfs/standard/var/pynode/lightning_info.py @@ -32,6 +32,7 @@ lightning_watchtower_server_info = None lightning_desync_count = 0 lightning_update_count = 0 +LIGHTNING_CACHE_FILE = "/tmp/lightning_info.json" LND_FOLDER = "/mnt/hdd/mynode/lnd/" TLS_CERT_FILE = "/mnt/hdd/mynode/lnd/tls.cert" LND_REST_PORT = "10080" @@ -92,6 +93,8 @@ def update_lightning_info(): if lightning_update_count < 30 or lightning_update_count % 2 == 0: update_lightning_tx_info() + update_lightning_json_cache() + return True def update_lightning_tx_info(): @@ -612,3 +615,22 @@ def enable_watchtower(): def disable_watchtower(): delete_file("/mnt/hdd/mynode/settings/.watchtower_enabled") + +# Only call from www process which has data +def update_lightning_json_cache(): + global LIGHTNING_CACHE_FILE + lightning_data = {} + lightning_data["info"] = get_lightning_info() + lightning_data["peers"] = get_lightning_peers() + lightning_data["channels"] = get_lightning_channels() + lightning_data["balances"] = get_lightning_balance_info() + #lightning_data["transactions"] = lightning_transactions + #lightning_data["payments"] = lightning_payments + #lightning_data["invoices"] = lightning_invoices + #lightning_data["watchtower_server_info"] = lightning_watchtower_server_info + return set_dictionary_file_cache(lightning_data, LIGHTNING_CACHE_FILE) + +# Can call from any process +def get_lightning_json_cache(): + global LIGHTNING_CACHE_FILE + return get_dictionary_file_cache(LIGHTNING_CACHE_FILE) \ No newline at end of file diff --git a/rootfs/standard/var/pynode/utilities.py b/rootfs/standard/var/pynode/utilities.py index 7dcd2d1d..599b1e9e 100644 --- a/rootfs/standard/var/pynode/utilities.py +++ b/rootfs/standard/var/pynode/utilities.py @@ -1,5 +1,7 @@ from flask import send_from_directory import os +import time +import json import subprocess import sys import codecs @@ -26,7 +28,6 @@ def to_bytes(s): def to_string(s): b = to_bytes(s) r = b.decode("utf-8") - print("S TYPE: "+str(type(r))) return r def quote_plus(s): @@ -85,6 +86,96 @@ def set_file_contents(filename, data): return False +#================================== +# Cache Functions +#================================== +utilities_cached_data = {} + +def is_cached(key, refresh_time=3600): # refresh=1hr + global utilities_cached_data + cache_time_key = key + "_cache_time" + now_time = int(time.time()) + if key in utilities_cached_data and cache_time_key in utilities_cached_data: + if utilities_cached_data[cache_time_key] + refresh_time < now_time: + return False + else: + return True + else: + return False + +def get_cached_data(key): + global utilities_cached_data + if key in utilities_cached_data: + return utilities_cached_data[key] + return None + +def update_cached_data(key, value): + global utilities_cached_data + cache_time_key = key + "_cache_time" + now_time = int(time.time()) + utilities_cached_data[key] = value + utilities_cached_data[cache_time_key] = now_time + +def set_dictionary_file_cache(data, file_path): + try: + with open(file_path, 'w') as file: + json.dump(data, file) + return True + except Exception as e: + log_message("ERROR set_dictionary_file_cache ({}):{} ".format(file_path, str(e))) + log_message(str(data)) + return False + +def get_dictionary_file_cache(file_path): + try: + with open(file_path) as file: + data = json.load(file) + return data + except Exception as e: + log_message("ERROR get_dictionary_file_cache ({}): {}".format(file_path, str(e))) + return None + +#================================== +# Settings File Functions +#================================== +def create_settings_file(name): + from drive_info import is_mynode_drive_mounted + + folder_1="/home/bitcoin/.mynode/" + folder_2="/mnt/hdd/mynode/settings/" + path_1="{}{}".format(folder_1, name) + path_2="{}{}".format(folder_2, name) + touch(path_1) + if is_mynode_drive_mounted(): + touch(path_2) + +def delete_settings_file(name): + folder_1="/home/bitcoin/.mynode/" + folder_2="/mnt/hdd/mynode/settings/" + path_1="{}{}".format(folder_1, name) + path_2="{}{}".format(folder_2, name) + delete_file(path_1) + delete_file(path_2) + +def settings_file_exists(name): + from drive_info import is_mynode_drive_mounted + + folder_1="/home/bitcoin/.mynode/" + folder_2="/mnt/hdd/mynode/settings/" + path_1="{}{}".format(folder_1, name) + path_2="{}{}".format(folder_2, name) + if os.path.isfile(path_1) and os.path.isfile(path_2): + return True + elif os.path.isfile(path_1) or os.path.isfile(path_2): + # Make sure backup file is in place + touch(path_1) + if is_mynode_drive_mounted(): + touch(path_2) + return True + + return False + + #================================== # Logging Functions #================================== diff --git a/rootfs/standard/var/www/mynode/api.py b/rootfs/standard/var/www/mynode/api.py index 96b8bb8c..34426d01 100644 --- a/rootfs/standard/var/www/mynode/api.py +++ b/rootfs/standard/var/www/mynode/api.py @@ -9,6 +9,7 @@ from device_info import * from thread_functions import * from systemctl_info import * from application_info import * +from drive_info import * from price_info import * from messages import * if isPython3(): diff --git a/rootfs/standard/var/www/mynode/bitcoin.py b/rootfs/standard/var/www/mynode/bitcoin.py index f821ee3a..8cbec739 100644 --- a/rootfs/standard/var/www/mynode/bitcoin.py +++ b/rootfs/standard/var/www/mynode/bitcoin.py @@ -61,27 +61,7 @@ def bitcoin_status_page(): #blocks = blocks[:5] # Take top 5 # Peers - peers = [] - if peerdata != None: - for p in peerdata: - peer = p - - if "pingtime" in p: - peer["pingtime"] = int(p["pingtime"] * 1000) - else: - peer["pingtime"] = "N/A" - - if "bytessent" in p: - peer["tx"] = "{:.2f}".format(float(p["bytessent"]) / 1000 / 1000) - else: - peer["tx"] = "N/A" - - if "bytesrecv" in p: - peer["rx"] = "{:.2f}".format(float(p["bytesrecv"]) / 1000 / 1000) - else: - peer["rx"] = "N/A" - - peers.append(peer) + peers = peerdata # Local address local_address = "..." @@ -109,9 +89,9 @@ def bitcoin_status_page(): "block_num": info["blocks"], "header_num": info["headers"], "rpc_password": rpc_password, - "disk_size": (int(info["size_on_disk"]) / 1000 / 1000 / 1000), + "disk_usage": get_bitcoin_disk_usage(), "mempool_tx": mempool["size"], - "mempool_size": "{:.3} MB".format(float(mempool["bytes"]) / 1000 / 1000), + "mempool_size": "{:.2} MB".format(float(mempool["bytes"]) / 1000 / 1000), "is_testnet_enabled": is_testnet_enabled(), "wallets": walletdata, "bitcoin_whitepaper_exists": bitcoin_whitepaper_exists, diff --git a/rootfs/standard/var/www/mynode/mynode.py b/rootfs/standard/var/www/mynode/mynode.py index 5d963dba..8fde344b 100644 --- a/rootfs/standard/var/www/mynode/mynode.py +++ b/rootfs/standard/var/www/mynode/mynode.py @@ -31,6 +31,7 @@ from messages import get_message from thread_functions import * from datetime import timedelta from device_info import * +from drive_info import * from device_warnings import * from systemctl_info import * from application_info import * @@ -776,6 +777,9 @@ def start_threads(): dmesg_thread = BackgroundThread(monitor_dmesg, 60) # Runs forever, restart after 60 if it fails dmesg_thread.start() threads.append(dmesg_thread) + app_data_cache_thread = BackgroundThread(update_application_json_cache, 300) + app_data_cache_thread.start() + threads.append(app_data_cache_thread) app.logger.info("STARTED {} THREADS".format(len(threads))) diff --git a/rootfs/standard/var/www/mynode/premium_plus.py b/rootfs/standard/var/www/mynode/premium_plus.py index 6bec14bd..1b9efba4 100644 --- a/rootfs/standard/var/www/mynode/premium_plus.py +++ b/rootfs/standard/var/www/mynode/premium_plus.py @@ -2,6 +2,7 @@ from flask import Blueprint, render_template, session, abort, Markup, request, r from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException from pprint import pprint, pformat from device_info import * +from application_info import * from user_management import check_logged_in from enable_disable_functions import restart_service import json @@ -15,13 +16,25 @@ mynode_premium_plus = Blueprint('mynode_premium_plus',__name__) def premium_plus_page(): check_logged_in() + status = get_premium_plus_token_status() + status_text = "UNKNOWN" + if get_premium_plus_is_connected(): + status_text = "Connected!" + else: + if has_premium_plus_token(): + status_text = "Not Connected ({})".format(status) + else: + status_text = "Please Configure an Access Token" + # Load page templateData = { "title": "myNode Premium+", "has_access_token": has_premium_plus_token(), "access_token": get_premium_plus_token(), - "status": get_premium_plus_token_status(), + "premium_plus_settings": get_premium_plus_settings(), + "status": status, + "status_text": status_text, "is_connected": get_premium_plus_is_connected(), "last_sync": get_premium_plus_last_sync(), "ui_settings": read_ui_settings() @@ -31,8 +44,12 @@ def premium_plus_page(): @mynode_premium_plus.route("/premium-plus/sync") def premium_plus_sync_page(): check_logged_in() + + update_application_json_cache() restart_service("premium_plus_connect") - time.sleep(3) + restart_service("lnd_backup") + + time.sleep(8) flash("Syncing...", category="message") return redirect("/premium-plus") @@ -41,8 +58,10 @@ def premium_plus_clear_token_page(): check_logged_in() delete_premium_plus_token() reset_premium_plus_token_status() + restart_service("premium_plus_connect") - time.sleep(3) + + time.sleep(5) flash("Token Cleared", category="message") return redirect("/premium-plus") @@ -54,7 +73,37 @@ def premium_plus_set_token_page(): flash("Missing Token", category="error") return redirect("/premium-plus") save_premium_plus_token(token) + restart_service("premium_plus_connect") - time.sleep(3) + restart_service("lnd_backup") + + time.sleep(5) flash("Token Set", category="message") + return redirect("/premium-plus") + +@mynode_premium_plus.route("/premium-plus/toggle-setting") +def premium_plus_toggle_setting_page(): + check_logged_in() + name = request.args.get('name') + enabled = request.args.get('enabled') + if name == None or enabled == None: + flash("Missing Parameter", category="error") + return redirect("/premium-plus") + + # Save value + premium_plus_setting_names = get_premium_plus_setting_names() + if (name in premium_plus_setting_names): + if (enabled == "1"): + create_settings_file(name) + else: + delete_settings_file(name) + else: + flash("Unknown Setting", category="error") + return redirect("/premium-plus") + + restart_service("premium_plus_connect") + restart_service("lnd_backup") + + time.sleep(5) + flash("Setting Updated!", category="message") return redirect("/premium-plus") \ No newline at end of file diff --git a/rootfs/standard/var/www/mynode/templates/bitcoin.html b/rootfs/standard/var/www/mynode/templates/bitcoin.html index 0b881510..0687e9f7 100644 --- a/rootfs/standard/var/www/mynode/templates/bitcoin.html +++ b/rootfs/standard/var/www/mynode/templates/bitcoin.html @@ -149,7 +149,7 @@ Disk Usage - {{disk_size}} GB + {{disk_usage}} GB RPC Username diff --git a/rootfs/standard/var/www/mynode/templates/lnd.html b/rootfs/standard/var/www/mynode/templates/lnd.html index 3fb9d390..909631d0 100644 --- a/rootfs/standard/var/www/mynode/templates/lnd.html +++ b/rootfs/standard/var/www/mynode/templates/lnd.html @@ -280,7 +280,7 @@ {% endif %} - Watchtower + Watchtower Server