Upgrade Mempool to v2.1.2

* Add mempool v2

* Update to v2.1.2

* Stop tor re-install on each upgrade

* Add dates to changelog

* Upgrade Mempool to v2

CHANGELOG

@@ -1,4 +1,5 @@
 === v0.2.26 ===
+- Released: 02/13/21
 - Add CKBunker v0.9 (beta)
 - Add Sphinx Relay v1.3.8 (beta)
 - Upgrade Thunderhub to v0.12.7
@@ -7,9 +8,11 @@
 - Bug Fixes
 
 === v0.2.25 ===
+- Released: 02/05/21
 - Improve drive detection for Clone Tool
 
 === v0.2.24 ===
+- Released: 02/04/21
 - Upgrade Bitcoin to v0.21.0
 - Upgrade LND to v0.12.0
 - Upgrade JoinMarket to v0.8.0
@@ -19,6 +22,7 @@
 - Bug fixes and minor UI improvements
 
 === v0.2.23 ===
+- Released: 01/25/21
 - Update BTC RPC Explorer to v2.2.0
 - Use SSO for securing BTC RPC Explorer access
 - Move all Dojo files to external drive
@@ -26,6 +30,7 @@
 - Bug Fixes
 
 === v0.2.22 ===
+- Released: 01/16/21
 - Upgrade Caravan to v0.3.5
 - Fix issue with Mempool on HTTPS
 - Upgrade Specter to v1.0.0
@@ -34,6 +39,7 @@
 - Bug Fixes
 
 === v0.2.21 ===
+- Released: 01/02/21
 - Update Loop to v0.11.2
 - Upgrade RTL to v0.10.0
 - Upgrade Thunderhub to v0.12.2
@@ -44,6 +50,7 @@
 - Add Onion URL for Whirlpool
 
 === v0.2.20 ===
+- Released: 12/20/20
 - Upgrade Specter to v0.10.4
 - Upgrade Thunderhub to v0.11.0
 - Upgrade Whirlpool CLI to v0.10.9
@@ -51,6 +58,7 @@
 - Bug fixes
 
 === v0.2.19 ===
+- Released: 11/27/20
 - Add Pool
 - Upgrade electrs to v0.8.6
 - Upgrade Specter to v0.10.1

rootfs/standard/etc/nginx/sites-enabled/https_mempoolspace.conf

@@ -20,4 +20,11 @@ server {
         proxy_set_header Connection "Upgrade";
         proxy_set_header Host $host;
     }
+    location /api/v1/ws {
+        proxy_pass http://127.0.0.1:4080/api/v1/ws;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+        proxy_set_header Host $host;
+    }
 }

rootfs/standard/etc/systemd/system/mempoolspace.service

@@ -7,19 +7,26 @@ Wants=www.service docker_images.service
 After=www.service docker_images.service
 
 [Service]
+WorkingDirectory=/mnt/hdd/mynode/mempool
 EnvironmentFile=/mnt/hdd/mynode/settings/.btcrpc_environment
 ExecStartPre=/usr/bin/is_not_shutting_down.sh
 ExecStartPre=/usr/bin/wait_on_bitcoin.sh
 ExecStartPre=/usr/bin/wait_on_docker_image_install.sh
-ExecStart=/usr/bin/docker run --rm --name mempoolspace -p 4080:80 -e BITCOIN_NODE_HOST=172.17.0.1 -e BITCOIN_NODE_USER=mynode -e BITCOIN_NODE_PASS=${BTC_RPC_PASSWORD} -e MEMPOOL_REFRESH_RATE_MS=10000 mempoolspace
+ExecStart=/usr/local/bin/docker-compose up
-ExecStop=/usr/bin/docker stop -t 2 mempoolspace
+ExecStartPost=/usr/bin/mynode_post_mempool.sh
+
+ExecStop=/usr/local/bin/docker-compose down
+
+# Old v1
+#ExecStart=/usr/bin/docker run --rm --name mempoolspace -p 4080:80 -e BITCOIN_NODE_HOST=172.17.0.1 -e BITCOIN_NODE_USER=mynode -e BITCOIN_NODE_PASS=${BTC_RPC_PASSWORD} -e MEMPOOL_REFRESH_RATE_MS=10000 mempoolspace
+#ExecStop=/usr/bin/docker stop -t 2 mempoolspace
 
 User=root
 Group=root
 Type=simple
-TimeoutSec=240
+TimeoutSec=120
 Restart=always
-RestartSec=120
+RestartSec=30
 StandardOutput=syslog
 StandardError=syslog
 SyslogIdentifier=mempoolspace

rootfs/standard/usr/bin/mynode_docker_images.sh

@@ -48,33 +48,41 @@ while true; do
 
     # Upgrade mempool
     echo "Checking for new mempool..."
-    MEMPOOLSPACE_UPGRADE_VERSION=v1.0.1
+    MEMPOOL_UPGRADE_VERSION=v2.1.2
-    MEMPOOLSPACE_UPGRADE_URL=https://github.com/mempool/mempool/archive/${MEMPOOLSPACE_UPGRADE_VERSION}.zip
+    MEMPOOL_UPGRADE_URL=https://github.com/mempool/mempool/archive/${MEMPOOL_UPGRADE_VERSION}.tar.gz
-    MEMPOOLSPACE_UPGRADE_URL_FILE=/mnt/hdd/mynode/settings/mempoolspace_url
+    MEMPOOL_UPGRADE_URL_FILE=/mnt/hdd/mynode/settings/mempoolspace_url
     CURRENT=""
-    if [ -f $MEMPOOLSPACE_UPGRADE_URL_FILE ]; then
+    if [ -f $MEMPOOL_UPGRADE_URL_FILE ]; then
-        CURRENT=$(cat $MEMPOOLSPACE_UPGRADE_URL_FILE)
+        CURRENT=$(cat $MEMPOOL_UPGRADE_URL_FILE)
     fi
-    if [ "$CURRENT" != "$MEMPOOLSPACE_UPGRADE_URL" ]; then
+    if [ "$CURRENT" != "$MEMPOOL_UPGRADE_URL" ]; then
-        docker rmi mempoolspace || true
+        docker rmi mempoolspace || true     # Remove old v1 image
 
-        cd /opt/mynode
+        cd /mnt/hdd/mynode/mempool
-        rm -rf mempoolspace
+        mkdir -p data mysql/data mysql/db-scripts
-        wget $MEMPOOLSPACE_UPGRADE_URL -O mempool.zip
+        cp -f /usr/share/mynode/mempool-docker-compose.yml /mnt/hdd/mynode/docker-compose.yml
-        unzip -o mempool.zip
-        rm mempool.zip
-        mv mempool* mempoolspace
-        cd mempoolspace
-        sync
 
-        # myNode Hack - Force use of specific alpine image source
+        rm -rf /opt/download/mempool
-        sed -i "s/alpine:latest/alpine:3.12.3/g" Dockerfile
+        mkdir -p /opt/download/mempool
+        cd /opt/download/mempool
+        wget $MEMPOOL_UPGRADE_URL -O mempool.tar.gz
+        tar -xvf mempool.tar.gz
+        rm mempool.tar.gz
+        mv mempool-* mempool
+        cp -f mempool/mariadb-structure.sql /mnt/hdd/mynode/mempool/mysql/db-scripts/mariadb-structure.sql
 
-        sleep 3s
+        # Update env variable to use latest version
-        docker build -t mempoolspace .
+        sed -i "s/VERSION=.*/VERSION=$MEMPOOL_UPGRADE_VERSION/g" /mnt/hdd/mynode/mempool/.env
-        if [ $? == 0 ]; then
+
-            echo $MEMPOOLSPACE_UPGRADE_URL > $MEMPOOLSPACE_UPGRADE_URL_FILE
+        docker pull mempool/frontend:${MEMPOOL_UPGRADE_VERSION}
+        docker pull mempool/backend:${MEMPOOL_UPGRADE_VERSION}
+
+        enabled=$(systemctl is-enabled mempoolspace)
+        if [ "$enabled" = "enabled" ]; then
+            systemctl restart mempoolspace &
         fi
+
+        echo $MEMPOOL_UPGRADE_URL > $MEMPOOL_UPGRADE_URL_FILE
     fi
 
     # Install Dojo
@@ -145,6 +153,7 @@ while true; do
     fi
 
     rm -f /tmp/installing_docker_images
+    touch /tmp/installing_docker_images_completed_once
 
     # Wait a day
     sleep 1d

rootfs/standard/usr/bin/mynode_post_mempool.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -x
+
+source /usr/share/mynode/mynode_config.sh
+
+sleep 10s
+
+# initalize mysql db (REQUIRED TO START MYSQL)
+isRunning=""
+
+# check on loop if  mysql db is running. when running initialize
+while [ 1 ]; do
+  # Check if mempool mysql db is running (check the db container)
+  isRunning=$(docker inspect --format="{{.State.Running}}" mempool_db_1)
+  if [ "$isRunning" == "true" ]; then
+    sleep 5s
+    blocks=$(docker exec -i mempool_db_1 mysql -uroot -padmin -D mempool -e "show tables;" | grep blocks)
+    if [[ "$blocks" == *"blocks"* ]]; then
+        echo "Mempool DB initialized!"
+        exit 0;
+    fi
+    if [ $IS_RASPI == 1 ]; then
+        echo "Initializing mempool db..."
+        docker exec -i mempool_db_1 bash -c "mysql -u root -padmin mempool" </mnt/hdd/mynode/mempool/mysql/db-scripts/mariadb-structure.sql
+        if [ $? -eq 0 ]; then
+            echo "Import success. Restart service by exiting 1."
+            exit 1
+        fi
+    fi
+  else
+    echo "Waiting to initialize mempool DB..."
+    sleep 10s
+  fi
+done
+
+exit 0

rootfs/standard/usr/bin/mynode_post_upgrade.sh

@@ -767,12 +767,12 @@ fi
 
 
 # Upgrade Tor
-rm -f /usr/local/bin/tor || true
+#rm -f /usr/local/bin/tor || true
-TOR_VERSION=$(tor --version)
+#TOR_VERSION=$(tor --version)
-if [[ "$TOR_VERSION" != *"Tor version 0.4"* ]]; then
+#if [[ "$TOR_VERSION" != *"Tor version 0.4"* ]]; then
-    $TORIFY apt-get remove -y tor
+#    $TORIFY apt-get remove -y tor
-    $TORIFY apt-get install -y tor
+#    $TORIFY apt-get install -y tor
-fi
+#fi
 
 
 # Enable fan control

rootfs/standard/usr/bin/mynode_reinstall_app.sh

@@ -45,7 +45,8 @@ elif [ "$APP" = "netdata" ]; then
 elif [ "$APP" = "mempoolspace" ]; then
     rm -f /mnt/hdd/mynode/settings/mempoolspace_url
     systemctl stop mempoolspace
-    docker rmi mempoolspace
+    docker rmi mempoolspace || true
+    rm -rf /mnt/hdd/mynode/mempool/*
 elif [ "$APP" = "btcpayserver" ]; then
     . "/opt/mynode/btcpayserver/btcpay-env.sh" && cd "$BTCPAY_BASE_DIRECTORY" && . helpers.sh && btcpay_remove
     cd ~

rootfs/standard/usr/bin/mynode_startup.sh

@@ -161,6 +161,7 @@ mkdir -p /mnt/hdd/mynode/specter
 mkdir -p /mnt/hdd/mynode/ckbunker
 mkdir -p /mnt/hdd/mynode/sphinxrelay
 mkdir -p /mnt/hdd/mynode/joinmarket
+mkdir -p /mnt/hdd/mynode/mempool
 mkdir -p /tmp/flask_uploads
 echo "drive_mounted" > $MYNODE_STATUS_FILE
 chmod 777 $MYNODE_STATUS_FILE
@@ -421,6 +422,15 @@ if [ ! -f /mnt/hdd/mynode/joinmarket/joinmarket.cfg ]; then
 fi
 chown -R joinmarket:joinmarket /mnt/hdd/mynode/joinmarket
 
+# Setup Mempool
+cp -f /usr/share/mynode/mempool-docker-compose.yml /mnt/hdd/mynode/mempool/docker-compose.yml
+if [ ! -f /mnt/hdd/mynode/mempool/.env ]; then
+    cp -f /usr/share/mynode/mempool.env /mnt/hdd/mynode/mempool/.env
+fi
+if [ $IS_RASPI -eq 1 ]; then
+    sed -i "s|MARIA_DB_IMAGE=.*|MARIA_DB_IMAGE=hypriot/rpi-mysql:latest|g" /mnt/hdd/mynode/mempool/.env
+fi
+
 # Setup udev
 chown root:root /etc/udev/rules.d/* || true
 udevadm trigger
@@ -441,6 +451,9 @@ fi
 if [ -f /mnt/hdd/mynode/joinmarket/joinmarket.cfg ]; then
     sed -i "s/rpc_password = .*/rpc_password = $BTCRPCPW/g" /mnt/hdd/mynode/joinmarket/joinmarket.cfg
 fi
+if [ -f /mnt/hdd/mynode/mempool/.env ]; then
+    sed -i "s/BITCOIN_RPC_PASS=.*/BITCOIN_RPC_PASS=$BTCRPCPW/g" /mnt/hdd/mynode/mempool/.env
+fi
 echo "BTC_RPC_PASSWORD=$BTCRPCPW" > /mnt/hdd/mynode/settings/.btcrpc_environment
 chown bitcoin:bitcoin /mnt/hdd/mynode/settings/.btcrpc_environment
 if [ -f /mnt/hdd/mynode/bitcoin/bitcoin.conf ]; then
@@ -450,6 +463,13 @@ cp -f /mnt/hdd/mynode/bitcoin/bitcoin.conf /home/admin/.bitcoin/bitcoin.conf
 chown admin:admin /home/admin/.bitcoin/bitcoin.conf
 
 
+# Append bitcoin UID and GID to btcrpc_environment
+BITCOIN_UID=$(id -u bitcoin)
+BITCOIN_GID=$(id -g bitcoin)
+echo "BITCOIN_UID=$BITCOIN_UID" >> /mnt/hdd/mynode/settings/.btcrpc_environment
+echo "BITCOIN_GID=$BITCOIN_GID" >> /mnt/hdd/mynode/settings/.btcrpc_environment
+
+
 # Reset BTCARGS
 echo "BTCARGS=" > /mnt/hdd/mynode/bitcoin/env
 

rootfs/standard/usr/bin/wait_on_docker_image_install.sh

@@ -2,8 +2,8 @@
 
 set -x
 
-# Wait a few second to give marker file a change to get created
+# Wait a few second to give marker file a chance to get created
-sleep 10s
+sleep 15s
 
 # Check if 
 echo "Checking if docker images have been installed..."

rootfs/standard/usr/share/mynode/mempool-docker-compose.yml

@@ -0,0 +1,47 @@
+version: "3.7"
+
+services:
+  web:
+    image: mempool/frontend:${VERSION}
+    restart: on-failure
+    stop_grace_period: 1m
+    command: "./wait-for db:3306 --timeout=720 -- nginx -g 'daemon off;'"
+    ports:
+      - 4080:8080
+    environment:
+      FRONTEND_HTTP_PORT: "8080"
+      BACKEND_MAINNET_HTTP_HOST: "api"
+  api:
+    image: mempool/backend:${VERSION}
+    user: 0:0
+    restart: on-failure
+    stop_grace_period: 1m
+    command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"
+    volumes:
+      - /mnt/hdd/mynode/mempool/data:/backend/cache
+    environment:
+      RPC_HOST: "172.17.0.1"
+      RPC_PORT: "8332"
+      RPC_USER: "mynode"
+      RPC_PASS: "${BITCOIN_RPC_PASS}"
+      ELECTRS_HOST: "172.17.0.1"
+      ELECTRS_PORT: "50001"
+      MYSQL_HOST: "db"
+      MYSQL_PORT: "3306"
+      MYSQL_DATABASE: "mempool"
+      MYSQL_USER: "mempool"
+      MYSQL_PASS: "mempool"
+      BACKEND_MAINNET_HTTP_PORT: "8999"
+      CACHE_DIR: "/backend/cache/"
+  db:
+    image: ${MARIA_DB_IMAGE}
+    restart: on-failure
+    stop_grace_period: 1m
+    volumes:
+      - /mnt/hdd/mynode/mempool/mysql/data:/var/lib/mysql
+      - /mnt/hdd/mynode/mempool/mysql/db-scripts:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_DATABASE: "mempool"
+      MYSQL_USER: "mempool"
+      MYSQL_PASSWORD: "mempool"
+      MYSQL_ROOT_PASSWORD: "admin"

rootfs/standard/usr/share/mynode/mempool.env

@@ -0,0 +1,3 @@
+VERSION=FILL_IN
+BITCOIN_RPC_PASS=FILL_IN
+MARIA_DB_IMAGE=mariadb:10.5.8