More Premium+; Python refactor

rootfs/standard/usr/bin/mynode_backup_scb.py

@@ -26,7 +26,13 @@ log.setLevel(logging.INFO)
 set_logger(log)
 
 # Helper functions
-
+cached_remote_hash = "NONE"
+def get_saved_remote_backup_hash():
+    global cached_remote_hash
+    return cached_remote_hash
+def set_saved_remote_backup_hash(hash):
+    global cached_remote_hash
+    cached_remote_hash = hash
 
 # Local Backup
 def local_backup(original_scb, backup_scb):
@@ -51,21 +57,40 @@ def local_backup(original_scb, backup_scb):
 # Remote Backup
 def remote_backup(original, backup):
 
+    # Check if remote backup is enabled
+    premium_plus_settings = get_premium_plus_settings()
+    if not premium_plus_settings['backup_scb']:
+        log_message("Remote Backup: SCB Backup Disabled")
+        return
+
     # Mainnet only
     if is_testnet_enabled():
-        log_message("Remote Backup: Skipping (testnet enabled")
+        log_message("Remote Backup: Skipping (testnet enabled)")
         return
 
     # Premium+ Feature
-    if not has_premium_plus_token() or get_premium_plus_token_status != "OK":
+    if not has_premium_plus_token() or get_premium_plus_token_status() != "OK":
         log_message("Remote Backup: Skipping (not Premium+)")
         return
 
+    md5_1 = get_md5_file_hash(original_scb)
+    md5_2 = get_saved_remote_backup_hash()
+    log_message("  Hash 1: {}".format(md5_1))
+    log_message("  Hash 2: {}".format(md5_2))
+    if md5_1 == md5_2:
+        log_message("Remote Backup: Hashes Match. Skipping Backup.")
+        return
+
     # POST Data
+    try:
+        file_data = {'scb': open(original,'rb')}
+    except Exception as e:
+        log_message("Remote Backup: Error reading SCB file.")
+        return
+
     data = {
         "token": get_premium_plus_token(),
-        "product_key": get_product_key(),
-        "scb_file": "REPLACE ME WITH FILE CONTENTS"
+        "product_key": get_product_key()
     }
 
     # Setup tor proxy
@@ -82,13 +107,14 @@ def remote_backup(original, backup):
             # Use tor for check in unless there have been tor 5 failures in a row
             r = None
             if (fail_count+1) % 5 == 0:
-                r = requests.post(BACKUP_SCB_URL, data=data, timeout=20)
+                r = requests.post(BACKUP_SCB_URL, data=data, files=file_data, timeout=20)
             else:
-                r = session.post(BACKUP_SCB_URL, data=data, timeout=20)
+                r = session.post(BACKUP_SCB_URL, data=data, files=file_data, timeout=20)
             
             if r.status_code == 200:
                 if r.text == "OK":
                     log_message("Remote Backup: Success ({})".format(r.text))
+                    set_saved_remote_backup_hash( md5_1 )
                 else:
                     log_message("Remote Backup: Error: ({})".format(r.text))
                 backup_success = True

rootfs/standard/usr/bin/mynode_post_upgrade.sh

@@ -208,7 +208,7 @@ if ! skip_base_upgrades ; then
     # Remove old python files so new copies are used (files migrated to pynode)
     set +x
     PYNODE_FILES="/var/pynode/*.py"
-    for pynode_file in $PYNODE_FILES
+    for pynode_file in $PYNODE_FILES; do
         echo "Migrating pynode file $pynode_file..."
         pynode_file="$(basename -- $pynode_file)"
         rm -f /var/www/mynode/${pynode_file}    # .py

rootfs/standard/usr/bin/mynode_premium_plus_connect.py

@@ -9,6 +9,9 @@ from systemd import journal
 from utilities import *
 from device_info import *
 from drive_info import *
+from application_info import *
+from bitcoin_info import *
+from lightning_info import *
 
 
 PREMIUM_PLUS_CONNECT_URL = "https://www.mynodebtc.com/device_api/premium_plus_connect.php"
@@ -19,7 +22,46 @@ log.setLevel(logging.INFO)
 set_logger(log)
 
 # Helper functions
+def get_premium_plus_device_info():
+    info = {}
+    settings = get_premium_plus_settings()
 
+    # Basic Info
+    info["serial"] = get_device_serial()
+    info["device_type"] = get_device_type()
+    info["device_arch"] = get_device_arch()
+    info["drive_size"] = get_mynode_drive_size()
+    info["data_drive_usage"] = get_data_drive_usage()
+    info["os_drive_usage"] = get_os_drive_usage()
+    info["temperature"] = get_device_temp()
+    info["total_ram"] = get_device_ram()
+    info["ram_usage"] = get_ram_usage()
+    info["swap_usage"] = get_swap_usage()
+    info["uptime"] = get_system_uptime()
+
+    # App status info
+    if settings['sync_status']:
+        info["app_info"] = get_all_applications_from_json_cache()
+
+    return info
+
+def get_premium_plus_bitcoin_info():
+    info = {}
+    settings = get_premium_plus_settings()
+
+    if settings['sync_bitcoin_and_lightning']:
+        log_message("BITCOIN " + str(get_bitcoin_json_cache()))
+        info = get_bitcoin_json_cache()
+    return info
+
+def get_premium_plus_lightning_info():
+    info = {}
+    settings = get_premium_plus_settings()
+
+    if settings['sync_bitcoin_and_lightning']:
+        log_message("LIGHTNING " + str(get_lightning_json_cache()))
+        info = get_lightning_json_cache()
+    return info
 
 # Update hourly
 def premium_plus_connect():
@@ -27,13 +69,13 @@ def premium_plus_connect():
     # Check in
     data = {
         "serial": get_device_serial(),
-        "device_type": get_device_type(),
-        "device_arch": get_device_arch(),
         "version": get_current_version(),
         "token": get_premium_plus_token(),
+        "settings": json.dumps(get_premium_plus_settings()),
+        "device_info": json.dumps(get_premium_plus_device_info()),
+        "bitcoin_info": json.dumps(get_premium_plus_bitcoin_info()),
+        "lightning_info": json.dumps(get_premium_plus_lightning_info()),
         "product_key": get_product_key(),
-        "drive_size": get_mynode_drive_size(),
-        "drive_usage": get_mynode_drive_usage(),
     }
 
     # Setup tor proxy
@@ -105,7 +147,7 @@ if __name__ == "__main__":
             wd = inotify.add_watch('/home/bitcoin/.mynode/', watch_flags)
             for event in inotify.read(timeout=one_hour_in_ms):
                 log_message("File changed: " + str(event))
-            log_message("Running connect again: " + str(event))
+            log_message("Running connect again")
         except Exception as e:
             log_message("Error: {}".format(e))
             time.sleep(60)

rootfs/standard/usr/share/btcpayserver/docker-compose.generated.yml

@@ -16,6 +16,7 @@ services:
       BTCPAY_SSHKEYFILE: ${BTCPAY_SSHKEYFILE}
       BTCPAY_SSHAUTHORIZEDKEYS: ${BTCPAY_SSHAUTHORIZEDKEYS}
       BTCPAY_DEBUGLOG: btcpay.log
+      BTCPAY_DOCKERDEPLOYMENT: "true"
       BTCPAY_CHAINS: "btc"
       BTCPAY_BTCEXPLORERURL: http://localhost:32838/
       BTCPAY_BTCLIGHTNING: "type=lnd-rest;server=https://localhost:10080/;macaroonfilepath=/etc/lnd_bitcoin/data/chain/bitcoin/mainnet/admin.macaroon;allowinsecure=true"
@@ -36,6 +37,7 @@ services:
     - "/mnt/hdd/mynode/lnd:/etc/lnd_bitcoin"
     - "/root/.ssh/authorized_keys:${BTCPAY_SSHAUTHORIZEDKEYS}"
     - "/root/.ssh/id_rsa_btcpay:${BTCPAY_SSHKEYFILE}"
+    - "btcpay_pluginsdir:/root/.btcpayserver/Plugins"
   nbxplorer:
     restart: unless-stopped
     image: nicolasdorier/nbxplorer:${NBXPLORER_VERSION}
@@ -45,17 +47,23 @@ services:
     environment:
       NBXPLORER_NETWORK: ${NBITCOIN_NETWORK:-regtest}
       NBXPLORER_BIND: 0.0.0.0:32838
+      NBXPLORER_TRIMEVENTS: 10000
       NBXPLORER_SIGNALFILESDIR: /datadir
       NBXPLORER_CHAINS: "btc"
       NBXPLORER_BTCRPCURL: http://localhost:8332/
       NBXPLORER_BTCNODEENDPOINT: localhost:8333
     volumes:
     - "nbxplorer_datadir:/datadir"
-    - "/mnt/hdd/mynode/bitcoin:/root/.bitcoin"
+
+
+
+
   postgres:
     restart: unless-stopped
-    image: postgres:${POSTGRES_VERSION}
+    image: btcpayserver/postgres:${POSTGRES_VERSION}
     network_mode: host
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
     volumes:
     - "postgres_datadir:/var/lib/postgresql/data"
 volumes:

rootfs/standard/usr/share/btcpayserver/env

@@ -24,4 +24,4 @@ TOR_RELAY_EMAIL=
 # myNode Additions
 BTCPAY_VERSION=REPLACE_BTCPAY_VERSION
 NBXPLORER_VERSION=REPLACE_NBXPLORER_VERSION
-POSTGRES_VERSION=9.6.5
+POSTGRES_VERSION=13.4.4

rootfs/standard/usr/share/mynode/application_info.json

@@ -288,7 +288,6 @@
         "comment": "HIDDEN FOR NOW - Enable show_on_homepage",
         "name": "Premium+",
         "short_name": "premium_plus",
-        "hide_status_icon": true,
         "can_enable_disable": false,
         "app_tile_button_text": "Manage",
         "app_tile_default_status_text": "Access and Backup",

rootfs/standard/usr/share/mynode/lnd.conf

@@ -50,4 +50,7 @@ bitcoind.rpchost=127.0.0.1:8332
 db.bolt.auto-compact=true
 
 [routing]
-routing.strictgraphpruning=true
+routing.strictgraphpruning=true
+
+[Wtclient]
+wtclient.active=1

rootfs/standard/usr/share/mynode/lnd_watchtower.conf

@@ -2,5 +2,3 @@
 [Watchtower]
 watchtower.active=1
 
-[Wtclient]
-wtclient.active=1

rootfs/standard/var/pynode/application_info.py

@@ -12,6 +12,7 @@ import re
 import os
 
 # Cached data
+JSON_APPLICATION_CACHE_FILE = "/tmp/app_cache.json"
 mynode_applications = None
 
 # Utility functions
@@ -118,8 +119,8 @@ def initialize_application_defaults(app):
     if not "show_on_application_page" in app: app["show_on_application_page"] = True
     if not "can_enable_disable" in app: app["can_enable_disable"] = True
     if not "is_enabled" in app: app["is_enabled"] = is_service_enabled( app["short_name"] )
-    #app["status"] = status # Should status be optional to include? Takes lots of time.
-    #app["status_color"] = get_service_status_color(short_name)
+    #app["status"] = get_application_status( app["short_name"] )
+    #app["status_color"] = get_service_status_color( app["short_name"] )
     if not "hide_status_icon" in app: app["hide_status_icon"] = False
     if not "log_file" in app: app["log_file"] = get_application_log_file( app["short_name"] )
     if not "journalctl_log_name" in app: app["journalctl_log_name"] = None
@@ -134,11 +135,12 @@ def initialize_application_defaults(app):
 
     return app
 
-def update_application(app):
+def update_application(app, include_status=False):
     short_name = app["short_name"]
     app["is_enabled"] = is_service_enabled(short_name)
-    #app["status"] = "???" # Should status be optional to include? Takes lots of time.
-    #app["status_color"] = get_service_status_color(short_name)
+    if include_status:
+        app["status"] = get_application_status( app["short_name"] )
+        app["status_color"] = get_service_status_color( app["short_name"] )
 
 def initialize_applications():
     global mynode_applications
@@ -157,11 +159,11 @@ def initialize_applications():
         mynode_applications = copy.deepcopy(apps)
     return
 
-def update_applications():
+def update_applications(include_status=False):
     global mynode_applications
 
     for app in mynode_applications:
-        update_application(app)
+        update_application(app, include_status)
 
 def clear_application_cache():
     global mynode_applications
@@ -180,7 +182,7 @@ def need_application_refresh():
         return True
     return False
 
-def get_all_applications(order_by="none"):
+def get_all_applications(order_by="none", include_status=False):
     global mynode_applications
 
     if need_application_refresh():
@@ -188,6 +190,9 @@ def get_all_applications(order_by="none"):
     else:
         update_applications()
 
+    if include_status:
+        update_applications(include_status)
+
     apps = copy.deepcopy(mynode_applications)
     if order_by == "alphabetic":
         apps.sort(key=lambda x: x["name"])
@@ -196,6 +201,17 @@ def get_all_applications(order_by="none"):
 
     return apps
 
+# Only call this from the www python process so status data is available
+def update_application_json_cache():
+    global JSON_APPLICATION_CACHE_FILE
+    apps = get_all_applications(order_by="alphabetic", include_status=True)
+    return set_dictionary_file_cache(apps, JSON_APPLICATION_CACHE_FILE)
+
+# Getting the data can be called from any process
+def get_all_applications_from_json_cache():
+    global JSON_APPLICATION_CACHE_FILE
+    return get_dictionary_file_cache(JSON_APPLICATION_CACHE_FILE)
+
 def get_application(short_name):
     apps = get_all_applications()
     for app in apps:
@@ -310,6 +326,14 @@ def get_application_status_color_special(short_name):
             dojo_initialized = ""
         if dojo_initialized != "true":
             return "red"
+    elif short_name == "premium_plus":
+        if has_premium_plus_token():
+            if get_premium_plus_is_connected():
+                return "green"
+            else:
+                return "red"
+        else:
+            return "gray"
     return ""
 
 def get_application_status_color(short_name):

rootfs/standard/var/pynode/bitcoin_info.py

@@ -22,6 +22,7 @@ bitcoin_wallets = None
 bitcoin_mempool = None
 bitcoin_recommended_fees = None
 bitcoin_version = None
+BITCOIN_CACHE_FILE = "/tmp/bitcoin_info.json"
 
 # Functions
 def get_bitcoin_rpc_username():
@@ -57,15 +58,24 @@ def update_bitcoin_main_info():
         rpc_connection = AuthServiceProxy("http://%s:%s@127.0.0.1:8332"%(rpc_user, rpc_pass), timeout=120)
 
         # Basic Info
-        bitcoin_blockchain_info = rpc_connection.getblockchaininfo()
-        if bitcoin_blockchain_info != None:
-            bitcoin_block_height = bitcoin_blockchain_info['headers']
-            mynode_block_height = bitcoin_blockchain_info['blocks']
+        info = rpc_connection.getblockchaininfo()
+        if info != None:
+            # Save specific data
+            bitcoin_block_height = info['headers']
+            mynode_block_height = info['blocks']
+            # Data cleanup
+            if "difficulty" in info:
+                info["difficulty"] = "{:.3g}".format(info["difficulty"])
+            if "verificationprogress" in info:
+                info["verificationprogress"] = "{:.3g}".format(info["verificationprogress"])
+
+        bitcoin_blockchain_info = info
 
     except Exception as e:
         log_message("ERROR: In update_bitcoin_info - {}".format( str(e) ))
         return False
 
+    update_bitcoin_json_cache()
     return True
 
 def update_bitcoin_other_info():
@@ -99,14 +109,37 @@ def update_bitcoin_other_info():
                 bitcoin_recent_blocks = rpc_connection.batch_([ [ "getblock", h ] for h in block_hashes ])
                 bitcoin_recent_blocks_last_cache_height = mynode_block_height
 
-            # Get peers
-            bitcoin_peers = rpc_connection.getpeerinfo()
+            # Get peers and cleanup data
+            peerdata = rpc_connection.getpeerinfo()
+            peers = []
+            if peerdata != None:
+                for p in peerdata:
+                    peer = p
+
+                    peer["pingtime"] = int(p["pingtime"] * 1000) if ("pingtime" in p) else "N/A"
+                    peer["tx"] = "{:.2f}".format(float(p["bytessent"]) / 1000 / 1000) if ("bytessent" in p) else "N/A"
+                    peer["rx"] = "{:.2f}".format(float(p["bytesrecv"]) / 1000 / 1000) if ("bytesrecv" in p) else "N/A"
+                    peer["minping"] = str(p["minping"]) if ("minping" in p) else "N/A"
+                    peer["minfeefilter"] = str(p["minfeefilter"]) if ("minfeefilter" in p) else "N/A"
+                    peer["pingwait"] = str(p["pingwait"]) if ("pingwait" in p) else "N/A"
+
+                    peers.append(peer)
+            bitcoin_peers = peers
 
             # Get network info
-            bitcoin_network_info = rpc_connection.getnetworkinfo()
+            network_data = rpc_connection.getnetworkinfo()
+            if network_data != None:
+                network_data["relayfee"] = str(network_data["relayfee"])
+                network_data["incrementalfee"] = str(network_data["incrementalfee"])
+            bitcoin_network_info = network_data
 
             # Get mempool
-            bitcoin_mempool = rpc_connection.getmempoolinfo()
+            mempool_data = rpc_connection.getmempoolinfo()
+            if mempool_data != None:
+                mempool_data["total_fee"] = str(mempool_data["total_fee"])
+                mempool_data["mempoolminfee"] = str(mempool_data["mempoolminfee"])
+                mempool_data["minrelaytxfee"] = str(mempool_data["minrelaytxfee"])
+            bitcoin_mempool = mempool_data
 
             # Get wallet info
             wallets = rpc_connection.listwallets()
@@ -145,6 +178,7 @@ def update_bitcoin_other_info():
         log_message("ERROR: In update_bitcoin_other_info - {}".format( str(e2) ))
         return False
 
+    update_bitcoin_json_cache()
     return True
 
 def get_bitcoin_status():
@@ -169,7 +203,7 @@ def get_bitcoin_blockchain_info():
 def get_bitcoin_difficulty():
     info = get_bitcoin_blockchain_info()
     if "difficulty" in info:
-        return "{:.3g}".format(info["difficulty"])
+        return info["difficulty"]
     return "???"
 
 def get_bitcoin_block_height():
@@ -209,17 +243,27 @@ def get_bitcoin_mempool_info():
     mempool["size"] = "???"
     mempool["bytes"] = "0"
     if mempooldata != None:
+        mempool["display_size"] = "unknown"
         if "size" in mempooldata:
             mempool["size"] = mempooldata["size"]
         if "bytes" in mempooldata:
             mempool["bytes"] = mempooldata["bytes"]
+            mempool["display_bytes"] = "{:.1} MB".format( float(mempooldata["bytes"])/1000/1000 )
 
     return copy.deepcopy(mempool)
 
 def get_bitcoin_mempool_size():
     info = get_bitcoin_mempool_info()
     size = float(info["bytes"]) / 1000 / 1000
-    return "{:.3} MB".format(size)
+    return "{:.1} MB".format(size)
+
+def get_bitcoin_disk_usage():
+    info = get_bitcoin_blockchain_info()
+    if "size_on_disk" in info:
+        usage = int(info["size_on_disk"]) / 1000 / 1000 / 1000
+        return "{:.0f}".format(usage)
+    else:
+        return "UNK"
 
 def get_bitcoin_recommended_fees():
     global bitcoin_recommended_fees
@@ -297,4 +341,22 @@ def is_bip158_enabled():
 def enable_bip158():
     touch("/mnt/hdd/mynode/settings/.bip158_enabled")
 def disable_bip158():
-    delete_file("/mnt/hdd/mynode/settings/.bip158_enabled")
+    delete_file("/mnt/hdd/mynode/settings/.bip158_enabled")
+
+
+def update_bitcoin_json_cache():
+    global BITCOIN_CACHE_FILE
+    bitcoin_data = {}
+    bitcoin_data["current_block_height"] = mynode_block_height
+    bitcoin_data["blockchain_info"] = get_bitcoin_blockchain_info()
+    #bitcoin_data["recent_blocks"] = bitcoin_recent_blocks
+    bitcoin_data["peers"] = get_bitcoin_peers()
+    bitcoin_data["network_info"] = get_bitcoin_network_info()
+    bitcoin_data["mempool"] = get_bitcoin_mempool_info()
+    #bitcoin_data["recommended_fees"] = bitcoin_recommended_fees
+    bitcoin_data["disk_usage"] = get_bitcoin_disk_usage()
+    return set_dictionary_file_cache(bitcoin_data, BITCOIN_CACHE_FILE)
+
+def get_bitcoin_json_cache():
+    global BITCOIN_CACHE_FILE
+    return get_dictionary_file_cache(BITCOIN_CACHE_FILE)

rootfs/standard/var/pynode/device_info.py

@@ -12,6 +12,7 @@ import subprocess
 import random
 import string
 import re
+import psutil
 
 try:
     import qrcode
@@ -315,6 +316,44 @@ def get_device_ram():
     cached_data["ram"] = ram
     return ram
 
+def get_device_temp():
+    if is_cached("device_temp", 60):
+        return get_cached_data("device_temp")
+
+    device_temp = "..."
+    try:
+        results = to_string(subprocess.check_output("cat /sys/class/thermal/thermal_zone0/temp", shell=True))
+        temp = int(results) / 1000
+        device_temp = "{:.1f}".format(temp)
+        update_cached_data("device_temp", device_temp)
+    except:
+        return device_temp
+    return device_temp
+
+def get_ram_usage():
+    if is_cached("ram_usage", 120):
+        return get_cached_data("ram_usage")
+
+    ram_usage = "..."
+    try:
+        ram_info = psutil.virtual_memory()
+        ram_usage = "{:.1f}%".format(ram_info.percent)
+    except:
+        return ram_usage
+    return ram_usage
+    
+def get_swap_usage():
+    if is_cached("swap_usage", 120):
+        return get_cached_data("swap_usage")
+
+    swap_usage = "..."
+    try:
+        swap_info = psutil.swap_memory()
+        swap_usage = "{:.1f}%".format(swap_info.percent)
+    except:
+        return swap_usage
+    return swap_usage
+
 def get_local_ip():
     local_ip = "unknown"
     try:
@@ -795,6 +834,16 @@ def recheck_premium_plus_token():
     reset_premium_plus_token_status()
     os.system("systemctl restart premium_plus_connect")
 
+def get_premium_plus_setting_names():
+    return ["sync_status","sync_bitcoin_and_lightning","backup_scb","watchtower"]
+def get_premium_plus_settings():
+    names = get_premium_plus_setting_names()
+    settings = {}
+    for n in names:
+        settings[n] = False
+    for n in names:
+        settings[n] = settings_file_exists(n)
+    return settings
 
 #==================================
 # Drive Repair Functions

rootfs/standard/var/pynode/drive_info.py

@@ -52,8 +52,27 @@ def get_mynode_drive_size():
         size = -2
     return size
 
-def get_mynode_drive_usage():
-    return "TODO"
+def get_data_drive_usage():
+    if is_cached("data_drive_usage", 300):
+        return get_cached_data("data_drive_usage")
+    usage = "0%"
+    try:
+        usage = to_string(subprocess.check_output("df -h /mnt/hdd | grep /dev | awk '{print $5}'", shell=True))
+        update_cached_data("data_drive_usage", usage)
+    except:
+        return usage
+    return usage
+        
+def get_os_drive_usage():
+    if is_cached("os_drive_usage", 300):
+        return get_cached_data("os_drive_usage")
+    usage = "0%"
+    try:
+        usage = to_string(subprocess.check_output("df -h / | grep /dev | awk '{print $5}'", shell=True))
+        update_cached_data("os_drive_usage", usage)
+    except:
+        return usage
+    return usage
 
 def check_partition_for_mynode(partition):
     is_mynode = False

rootfs/standard/var/pynode/lightning_info.py

@@ -32,6 +32,7 @@ lightning_watchtower_server_info = None
 lightning_desync_count = 0
 lightning_update_count = 0
 
+LIGHTNING_CACHE_FILE = "/tmp/lightning_info.json"
 LND_FOLDER = "/mnt/hdd/mynode/lnd/"
 TLS_CERT_FILE = "/mnt/hdd/mynode/lnd/tls.cert"
 LND_REST_PORT = "10080"
@@ -92,6 +93,8 @@ def update_lightning_info():
         if lightning_update_count < 30 or lightning_update_count % 2 == 0:
             update_lightning_tx_info()
 
+    update_lightning_json_cache()
+
     return True
 
 def update_lightning_tx_info():
@@ -612,3 +615,22 @@ def enable_watchtower():
 
 def disable_watchtower():
     delete_file("/mnt/hdd/mynode/settings/.watchtower_enabled")
+
+# Only call from www process which has data
+def update_lightning_json_cache():
+    global LIGHTNING_CACHE_FILE
+    lightning_data = {}
+    lightning_data["info"] = get_lightning_info()
+    lightning_data["peers"] = get_lightning_peers()
+    lightning_data["channels"] = get_lightning_channels()
+    lightning_data["balances"] = get_lightning_balance_info()
+    #lightning_data["transactions"] = lightning_transactions
+    #lightning_data["payments"] = lightning_payments
+    #lightning_data["invoices"] = lightning_invoices
+    #lightning_data["watchtower_server_info"] = lightning_watchtower_server_info
+    return set_dictionary_file_cache(lightning_data, LIGHTNING_CACHE_FILE)
+
+# Can call from any process
+def get_lightning_json_cache():
+    global LIGHTNING_CACHE_FILE
+    return get_dictionary_file_cache(LIGHTNING_CACHE_FILE)

rootfs/standard/var/pynode/utilities.py

@@ -1,5 +1,7 @@
 from flask import send_from_directory
 import os
+import time
+import json
 import subprocess
 import sys
 import codecs
@@ -26,7 +28,6 @@ def to_bytes(s):
 def to_string(s):
     b = to_bytes(s)
     r = b.decode("utf-8")
-    print("S TYPE: "+str(type(r)))
     return r
 
 def quote_plus(s):
@@ -85,6 +86,96 @@ def set_file_contents(filename, data):
     return False
 
 
+#==================================
+# Cache Functions
+#==================================
+utilities_cached_data = {}
+
+def is_cached(key, refresh_time=3600): # refresh=1hr
+    global utilities_cached_data
+    cache_time_key = key + "_cache_time"
+    now_time = int(time.time())
+    if key in utilities_cached_data and cache_time_key in utilities_cached_data:
+        if utilities_cached_data[cache_time_key] + refresh_time < now_time:
+            return False
+        else:
+            return True
+    else:
+        return False
+
+def get_cached_data(key):
+    global utilities_cached_data
+    if key in utilities_cached_data:
+        return utilities_cached_data[key]
+    return None
+
+def update_cached_data(key, value):
+    global utilities_cached_data
+    cache_time_key = key + "_cache_time"
+    now_time = int(time.time())
+    utilities_cached_data[key] = value
+    utilities_cached_data[cache_time_key] = now_time
+
+def set_dictionary_file_cache(data, file_path):
+    try:
+        with open(file_path, 'w') as file:
+            json.dump(data, file)
+        return True
+    except Exception as e:
+        log_message("ERROR set_dictionary_file_cache ({}):{} ".format(file_path, str(e)))
+        log_message(str(data))
+        return False
+
+def get_dictionary_file_cache(file_path):
+    try:
+        with open(file_path) as file:
+            data = json.load(file)
+        return data
+    except Exception as e:
+        log_message("ERROR get_dictionary_file_cache ({}): {}".format(file_path, str(e)))
+        return None
+
+#==================================
+# Settings File Functions
+#==================================
+def create_settings_file(name):
+    from drive_info import is_mynode_drive_mounted
+
+    folder_1="/home/bitcoin/.mynode/"
+    folder_2="/mnt/hdd/mynode/settings/"
+    path_1="{}{}".format(folder_1, name)
+    path_2="{}{}".format(folder_2, name)
+    touch(path_1)
+    if is_mynode_drive_mounted():
+        touch(path_2)
+
+def delete_settings_file(name):
+    folder_1="/home/bitcoin/.mynode/"
+    folder_2="/mnt/hdd/mynode/settings/"
+    path_1="{}{}".format(folder_1, name)
+    path_2="{}{}".format(folder_2, name)
+    delete_file(path_1)
+    delete_file(path_2)
+
+def settings_file_exists(name):
+    from drive_info import is_mynode_drive_mounted
+
+    folder_1="/home/bitcoin/.mynode/"
+    folder_2="/mnt/hdd/mynode/settings/"
+    path_1="{}{}".format(folder_1, name)
+    path_2="{}{}".format(folder_2, name)
+    if os.path.isfile(path_1) and os.path.isfile(path_2):
+        return True
+    elif os.path.isfile(path_1) or os.path.isfile(path_2):
+        # Make sure backup file is in place
+        touch(path_1)
+        if is_mynode_drive_mounted():
+            touch(path_2)
+        return True
+    
+    return False
+
+
 #==================================
 # Logging Functions
 #==================================

rootfs/standard/var/www/mynode/api.py

@@ -9,6 +9,7 @@ from device_info import *
 from thread_functions import *
 from systemctl_info import *
 from application_info import *
+from drive_info import *
 from price_info import *
 from messages import *
 if isPython3():

rootfs/standard/var/www/mynode/bitcoin.py

@@ -61,27 +61,7 @@ def bitcoin_status_page():
             #blocks = blocks[:5] # Take top 5
 
         # Peers
-        peers = []
-        if peerdata != None:
-            for p in peerdata:
-                peer = p
-
-                if "pingtime" in p:
-                    peer["pingtime"] =  int(p["pingtime"] * 1000)
-                else:
-                    peer["pingtime"] = "N/A"
-
-                if "bytessent" in p:
-                    peer["tx"] = "{:.2f}".format(float(p["bytessent"]) / 1000 / 1000)
-                else:
-                    peer["tx"] = "N/A"
-
-                if "bytesrecv" in p:
-                    peer["rx"] = "{:.2f}".format(float(p["bytesrecv"]) / 1000 / 1000)
-                else:
-                    peer["rx"] = "N/A"
-
-                peers.append(peer)
+        peers = peerdata
 
         # Local address
         local_address = "..."
@@ -109,9 +89,9 @@ def bitcoin_status_page():
         "block_num": info["blocks"],
         "header_num": info["headers"],
         "rpc_password": rpc_password,
-        "disk_size": (int(info["size_on_disk"]) / 1000 / 1000 / 1000),
+        "disk_usage": get_bitcoin_disk_usage(),
         "mempool_tx": mempool["size"],
-        "mempool_size": "{:.3} MB".format(float(mempool["bytes"]) / 1000 / 1000),
+        "mempool_size": "{:.2} MB".format(float(mempool["bytes"]) / 1000 / 1000),
         "is_testnet_enabled": is_testnet_enabled(),
         "wallets": walletdata,
         "bitcoin_whitepaper_exists": bitcoin_whitepaper_exists,

rootfs/standard/var/www/mynode/mynode.py

@@ -31,6 +31,7 @@ from messages import get_message
 from thread_functions import *
 from datetime import timedelta
 from device_info import *
+from drive_info import *
 from device_warnings import *
 from systemctl_info import *
 from application_info import *
@@ -776,6 +777,9 @@ def start_threads():
     dmesg_thread = BackgroundThread(monitor_dmesg, 60) # Runs forever, restart after 60 if it fails 
     dmesg_thread.start()
     threads.append(dmesg_thread)
+    app_data_cache_thread = BackgroundThread(update_application_json_cache, 300)
+    app_data_cache_thread.start()
+    threads.append(app_data_cache_thread)
 
     app.logger.info("STARTED {} THREADS".format(len(threads)))
 

rootfs/standard/var/www/mynode/premium_plus.py

@@ -2,6 +2,7 @@ from flask import Blueprint, render_template, session, abort, Markup, request, r
 from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
 from pprint import pprint, pformat
 from device_info import *
+from application_info import *
 from user_management import check_logged_in
 from enable_disable_functions import restart_service
 import json
@@ -15,13 +16,25 @@ mynode_premium_plus = Blueprint('mynode_premium_plus',__name__)
 def premium_plus_page():
     check_logged_in()
 
+    status = get_premium_plus_token_status()
+    status_text = "UNKNOWN"
+    if get_premium_plus_is_connected():
+        status_text = "Connected!"
+    else:
+        if has_premium_plus_token():
+            status_text = "Not Connected ({})".format(status)
+        else:
+            status_text = "Please Configure an Access Token"
+
     
     # Load page
     templateData = {
         "title": "myNode Premium+",
         "has_access_token": has_premium_plus_token(),
         "access_token": get_premium_plus_token(),
-        "status": get_premium_plus_token_status(),
+        "premium_plus_settings": get_premium_plus_settings(),
+        "status": status,
+        "status_text": status_text,
         "is_connected": get_premium_plus_is_connected(),
         "last_sync": get_premium_plus_last_sync(),
         "ui_settings": read_ui_settings()
@@ -31,8 +44,12 @@ def premium_plus_page():
 @mynode_premium_plus.route("/premium-plus/sync")
 def premium_plus_sync_page():
     check_logged_in()
+
+    update_application_json_cache()
     restart_service("premium_plus_connect")
-    time.sleep(3)
+    restart_service("lnd_backup")
+
+    time.sleep(8)
     flash("Syncing...", category="message")
     return redirect("/premium-plus")
 
@@ -41,8 +58,10 @@ def premium_plus_clear_token_page():
     check_logged_in()
     delete_premium_plus_token()
     reset_premium_plus_token_status()
+
     restart_service("premium_plus_connect")
-    time.sleep(3)
+
+    time.sleep(5)
     flash("Token Cleared", category="message")
     return redirect("/premium-plus")
 
@@ -54,7 +73,37 @@ def premium_plus_set_token_page():
         flash("Missing Token", category="error")
         return redirect("/premium-plus")
     save_premium_plus_token(token)
+
     restart_service("premium_plus_connect")
-    time.sleep(3)
+    restart_service("lnd_backup")
+
+    time.sleep(5)
     flash("Token Set", category="message")
+    return redirect("/premium-plus")
+
+@mynode_premium_plus.route("/premium-plus/toggle-setting")
+def premium_plus_toggle_setting_page():
+    check_logged_in()
+    name = request.args.get('name')
+    enabled = request.args.get('enabled')
+    if name == None or enabled == None:
+        flash("Missing Parameter", category="error")
+        return redirect("/premium-plus")
+
+    # Save value
+    premium_plus_setting_names = get_premium_plus_setting_names()
+    if   (name in premium_plus_setting_names):
+        if (enabled == "1"):
+            create_settings_file(name)
+        else:
+            delete_settings_file(name)
+    else:
+        flash("Unknown Setting", category="error")
+        return redirect("/premium-plus")
+    
+    restart_service("premium_plus_connect")
+    restart_service("lnd_backup")
+
+    time.sleep(5)
+    flash("Setting Updated!", category="message")
     return redirect("/premium-plus")

rootfs/standard/var/www/mynode/templates/bitcoin.html

@@ -149,7 +149,7 @@
                         </tr>
                         <tr>
                             <th>Disk Usage</th>
-                            <td>{{disk_size}} GB</td>
+                            <td>{{disk_usage}} GB</td>
                         </tr>
                         <tr>
                             <th>RPC Username</th>

rootfs/standard/var/www/mynode/templates/lnd.html

@@ -280,7 +280,7 @@
                         </tr>
 {% endif %}
                         <tr>
-                            <th>Watchtower</th>
+                            <th>Watchtower Server</th>
                             <td>
                                 <label class="switch">
                                     <input type="checkbox" id="watchtower_enabled_checkbox" {% if watchtower_enabled %}checked{% endif %}>

rootfs/standard/var/www/mynode/templates/premium_plus.html

@@ -3,6 +3,25 @@
         <title>{{ title }}</title>
         {% include 'includes/head.html' %}
 
+        <style>
+            .ui-tooltip {
+                
+                font-size: 16px;
+                width: 400px;
+                max-width: 400px;
+                
+            }
+
+            {% if not is_connected %}
+            .connected-style {
+                color: #AAAAAA;
+            }
+            {% else %}
+            .connected-style {
+                
+            }
+            {% endif %}
+        </style>
 
         <script>
             $(document).ready(function() {
@@ -45,6 +64,86 @@
                     }
                 });
 
+                $("#sync_button").on("click", function() {
+                    $('#loading_spinner_overlay').fadeIn();
+                    window.location.href='/premium-plus/sync';
+                });
+
+
+                $('#sync_status_checkbox').change(function () {
+                    $("#sync_status_save").show();
+                });
+                $("#sync_status_save").on("click", function() {
+                    enabled=$('#sync_status_checkbox').is(":checked");
+                    if (enabled)
+                    {
+                        window.location.href='/premium-plus/toggle-setting?name=sync_status&enabled=1';
+                    }
+                    else
+                    {
+                        window.location.href='/premium-plus/toggle-setting?name=sync_status&enabled=0';
+                    }
+                });
+
+                $('#sync_bitcoin_and_lightning_checkbox').change(function () {
+                    $("#sync_bitcoin_and_lightning_save").show();
+                });
+                $("#sync_bitcoin_and_lightning_save").on("click", function() {
+                    enabled=$('#sync_bitcoin_and_lightning_checkbox').is(":checked");
+                    if (enabled)
+                    {
+                        window.location.href='/premium-plus/toggle-setting?name=sync_bitcoin_and_lightning&enabled=1';
+                    }
+                    else
+                    {
+                        window.location.href='/premium-plus/toggle-setting?name=sync_bitcoin_and_lightning&enabled=0';
+                    }
+                });
+
+                $('#backup_scb_checkbox').change(function () {
+                    $("#backup_scb_save").show();
+                });
+                $("#backup_scb_save").on("click", function() {
+                    enabled=$('#backup_scb_checkbox').is(":checked");
+                    if (enabled)
+                    {
+                        window.location.href='/premium-plus/toggle-setting?name=backup_scb&enabled=1';
+                    }
+                    else
+                    {
+                        window.location.href='/premium-plus/toggle-setting?name=backup_scb&enabled=0';
+                    }
+                });
+
+                $('#watchtower_checkbox').change(function () {
+                    $("#watchtower_save").show();
+                });
+                $("#watchtower_save").on("click", function() {
+                    enabled=$('#watchtower_checkbox').is(":checked");
+                    if (enabled)
+                    {
+                        window.location.href='/premium-plus/toggle-setting?name=watchtower&enabled=1';
+                    }
+                    else
+                    {
+                        window.location.href='/premium-plus/toggle-setting?name=watchtower&enabled=0';
+                    }
+                });
+
+                //$( document ).tooltip();
+
+                $( document ).tooltip({
+                    items: "[title]",
+                    content: function() {
+                        var element = $( this );
+                        if ( element.is( "[title]" ) ) {
+                            text =  "<b>"+element.attr( "title" )+"</b>"
+                            text += "<br/><br/>";
+                            text += ""+element.attr( "msg" )+"";
+                            return text;
+                        }
+                    }
+                });
             });
         </script>
      </head>
@@ -81,7 +180,7 @@
                         {% if has_access_token %}
                         <tr>
                             <th>Status</th>
-                            <td>{{status}}</td>
+                            <td>{{status_text}}</td>
                         </tr>
                         <tr>
                             <th>Last Sync</th>
@@ -89,7 +188,7 @@
                         </tr>
                         <tr>
                             <th>Actions</th>
-                            <td><a href="/premium-plus/sync" class="ui-button ui-widget ui-corner-all mynode_button_small">Force Sync</a></td>
+                            <td><button id="sync_button" class="ui-button ui-widget ui-corner-all mynode_button_small">Force Sync</button></td>
                         </tr>
                         {% endif %}
                     </table>
@@ -105,32 +204,64 @@
                 <div class="info_tile_contents">
                     <table class="info_table">
                         <tr>
-                            <th>Sync Device Status</th>
-                            <td>{{mempool_tx}}</td>
-                        </tr>
-                        <tr>
-                            <th>Backup Lightning SCB File</th>
-                            <td>{{mempool_tx}}</td>
-                        </tr>
-                        <tr>
-                            <th>myNode Watchtower</th>
-                            <td>{{mempool_size}}</td>
-                        </tr>
-                        <tr>
-                            <th>Proxy</th>
+                            <th class="connected-style">Sync Device Status</th>
                             <td>
-                                <p id="rpc_password" style="display: none; margin: 0;">{{rpc_password}} <span id="copy_rpc_password" class="ui-icon ui-icon-copy" style="cursor: pointer; display: none;" title="Copy Password"></span></p>
-                                <a id="show_rpc_password" class="ui-button ui-widget ui-corner-all mynode_button_small" style="width: 70%;">show</a>
+                                <label class="switch">
+                                    <input type="checkbox" id="sync_status_checkbox" {% if premium_plus_settings['sync_status'] %}checked{% endif %} {% if not is_connected %}disabled{% endif %}>
+                                    <span class="slider round"></span>
+                                </label>
+                                <button id="sync_status_save" style="display: none; float: right;" class="ui-button ui-widget ui-corner-all settings_button_small">Save</button>
+                            </td>
+                            <td>
+                                <img style="width: 16px; margin-left: 10px;" src="{{ url_for('static', filename="images/tooltip_icon.png")}}" title="Sync Device Status" msg="Syncing your node's status will allow you to see basic node information via mynodebtc.com. Info like app status, drive usage, and uptime will be available online.">
                             </td>
                         </tr>
                         <tr>
-                            <th>Peer Bloom Filters (BIP 37)</th>
+                            <th class="connected-style">Sync Bitcoin and Lightning Info</th>
                             <td>
                                 <label class="switch">
-                                    <input type="checkbox" id="bip37_checkbox" {% if bip37_enabled %}checked{% endif %}>
+                                    <input type="checkbox" id="sync_bitcoin_and_lightning_checkbox" {% if premium_plus_settings['sync_bitcoin_and_lightning'] %}checked{% endif %} {% if not is_connected %}disabled{% endif %}>
                                     <span class="slider round"></span>
                                 </label>
-                                <button id="bip37_save" style="display: none; float: right;" class="ui-button ui-widget ui-corner-all settings_button_small">Save</button>
+                                <button id="sync_bitcoin_and_lightning_save" style="display: none; float: right;" class="ui-button ui-widget ui-corner-all settings_button_small">Save</button>
+                            </td>
+                            <td>
+                                <img style="width: 16px; margin-left: 10px;" src="{{ url_for('static', filename="images/tooltip_icon.png")}}" title="Sync Bitcoin and Lightning Info" msg="Syncing your node's Bitcoin and Lightning information will allow you to view that information via mynodebtc.com. Bitcoin information (block height, peers, etc...) and Lightning information (peers, channels, balances, etc...) will be available online. Some information may be considered sensitive.">
+                            </td>
+                        </tr>
+                        <tr>
+                            <th class="connected-style">Remote Channel Backup</th>
+                            <td>
+                                <label class="switch">
+                                    <input type="checkbox" id="backup_scb_checkbox" {% if premium_plus_settings['backup_scb'] %}checked{% endif %} {% if not is_connected %}disabled{% endif %}>
+                                    <span class="slider round"></span>
+                                </label>
+                                <button id="backup_scb_save" style="display: none; float: right;" class="ui-button ui-widget ui-corner-all settings_button_small">Save</button>
+                            </td>
+                            <td>
+                                <img style="width: 16px; margin-left: 10px;" src="{{ url_for('static', filename="images/tooltip_icon.png")}}" title="Remote Channel Backup" msg="Enabling Remote Channel Backup creates another layer of defense. An online backup of your encrypted SCB file is made to help prevent loss of funds in case of physical hardware failure.">
+                            </td>
+                        </tr>
+                        <tr>
+                            <th class="connected-style">Lightning Watchtower</th>
+                            <td>
+                                <label class="switch">
+                                    <input type="checkbox" id="watchtower_checkbox" {% if premium_plus_settings['watchtower'] %}checked{% endif %} {% if not is_connected %}disabled{% endif %}>
+                                    <span class="slider round"></span>
+                                </label>
+                                <button id="watchtower_save" style="display: none; float: right;" class="ui-button ui-widget ui-corner-all settings_button_small">Save</button>
+                            </td>
+                            <td>
+                                <img style="width: 16px; margin-left: 10px;" src="{{ url_for('static', filename="images/tooltip_icon.png")}}" title="Lightning Watchtower" msg="Enabling the Lightning Watchtower starts an external node to monitor your Lightning channels, helping detect and prevent malicious actions by peers.">
+                            </td>
+                        </tr>
+                        <tr>
+                            <th class="connected-style">Proxy</th>
+                            <td class="connected-style">
+                                Coming soon!
+                            </td>
+                            <td>
+                                &nbsp;
                             </td>
                         </tr>
                     </table>
@@ -151,6 +282,12 @@
             </form>
         </div>
 
+        <div id="loading_spinner_overlay" class="loading_spinner_overlay" style="display:none;">
+            <img id="loading_spinner" class="loading_image" src="{{ url_for('static', filename="images/loading.gif")}}"/>
+            <br/>
+            Loading...
+        </div>
+
         {% include 'includes/footer.html' %}
      </body>
 </html>

rootfs/standard/var/www/mynode/thread_functions.py

@@ -16,79 +16,35 @@ import random
 
 # Info to get from the update threads
 has_updated_btc_info = False
-data_drive_usage = "0%"
-os_drive_usage = "0%"
 cpu_usage = "..."
 ram_usage = "..."
 swap_usage = "..."
-device_temp = "..."
 public_ip = "not_detected"
 
 # Getters
 def get_has_updated_btc_info():
     global has_updated_btc_info
     return has_updated_btc_info
-def get_data_drive_usage():
-    global data_drive_usage
-    return data_drive_usage
-def get_os_drive_usage():
-    global os_drive_usage
-    return os_drive_usage
 def get_cpu_usage():
     global cpu_usage
     return cpu_usage
-def get_ram_usage():
-    global ram_usage
-    return ram_usage
-def get_swap_usage():
-    global swap_usage
-    return swap_usage
-def get_device_temp():
-    global device_temp
-    return device_temp
 def get_public_ip():
     global public_ip
     return public_ip
 
 # Updates device info every 60 seconds
 def update_device_info():
-    global data_drive_usage
-    global os_drive_usage
     global cpu_usage
-    global ram_usage
-    global swap_usage
-    global device_temp
 
     # Get drive info
     try:
         # Get throttled info (raspi only)
         reload_throttled_data()
 
-        # Get drive actual usage
-        #results = to_string(subprocess.check_output(["du","-sh","/mnt/hdd/mynode/"]))
-        #drive_usage = results.split()[0]
-
-        # Get drive percent usage
-        data_drive_usage = to_string(subprocess.check_output("df -h /mnt/hdd | grep /dev | awk '{print $5}'", shell=True))
-        os_drive_usage = to_string(subprocess.check_output("df -h / | grep /dev | awk '{print $5}'", shell=True))
-
-        # Get RAM usage
-        ram_info = psutil.virtual_memory()
-        ram_usage = "{:.1f}%".format(ram_info.percent)
-
-        # Get Swap Usage
-        swap_info = psutil.swap_memory()
-        swap_usage = "{:.1f}%".format(swap_info.percent)
-
         # Get CPU usage
         cpu_info = psutil.cpu_times_percent(interval=10.0, percpu=False)
         cpu_usage = "{:.1f}%".format(100.0 - cpu_info.idle)
 
-        # Get device temp
-        results = to_string(subprocess.check_output("cat /sys/class/thermal/thermal_zone0/temp", shell=True))
-        temp = int(results) / 1000
-        device_temp = "{:.1f}".format(temp)
-
     except Exception as e:
         log_message("CAUGHT update_device_info EXCEPTION: " + str(e))
         return