CI: Migration SQLite to PostgreSQL (#719)

* migrate all extensions
* errors if migration is missing

.dockerignore

@@ -4,6 +4,7 @@ docker
 docs
 tests
 venv
+tools
 
 *.md
 *.log
@@ -12,7 +13,6 @@ venv
 
 .gitignore
 .prettierrc
-conv.py
 LICENSE
 Makefile
 mypy.ini

.github/workflows/migrations.yml

@@ -0,0 +1,49 @@
+name: migrations
+
+on: [pull_request]
+
+jobs:
+  sqlite-to-postgres:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          # maps tcp port 5432 on service container to the host
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      matrix:
+        python-version: [3.8]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        env: 
+          VIRTUAL_ENV: ./venv
+          PATH: ${{ env.VIRTUAL_ENV }}/bin:${{ env.PATH }}
+        run: |
+          python -m venv ${{ env.VIRTUAL_ENV }}
+          ./venv/bin/python -m pip install --upgrade pip
+          ./venv/bin/pip install -r requirements.txt
+          ./venv/bin/pip install pytest pytest-asyncio pytest-cov requests mock
+      - name: Run migrations
+        run: |
+          rm -rf ./data
+          mkdir -p ./data
+          export LNBITS_DATA_FOLDER="./data"
+          timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+          export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres"
+          timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+          ./venv/bin/python tools/conv.py --dont-ignore-missing

docs/guide/installation.md

@@ -142,8 +142,8 @@ LNBITS_DATABASE_URL="postgres://postgres:postgres@localhost/lnbits"
 
 # START LNbits
 # STOP LNbits
-# on the LNBits folder, locate and edit 'conv.py' with the relevant credentials
+# on the LNBits folder, locate and edit 'tools/conv.py' with the relevant credentials
-python3 conv.py
+python3 tools/conv.py
 ```
 
 Hopefully, everything works and get migrated... Launch LNbits again and check if everything is working properly.

lnbits/extensions/lnurldevice/migrations.py

@@ -42,7 +42,7 @@ async def m002_redux(db):
     """
     try:
         for row in [
-            list(row) for row in await db2.fetchall("SELECT * FROM lnurlpos.lnurlposs")
+            list(row) for row in await db2.fetchall("SELECT * FROM lnurlpos.lnurlpos")
         ]:
             await db.execute(
                 """

lnbits/extensions/lnurldevice/templates/lnurldevice/_api_docs.html

@@ -120,7 +120,7 @@
         <q-card-section>
           <code
             ><span class="text-blue">GET</span>
-            /lnurldevice/api/v1/lnurlposs</code
+            /lnurldevice/api/v1/lnurlpos</code
           >
           <h5 class="text-caption q-mt-sm q-mb-none">Headers</h5>
           <code>{"X-Api-Key": &lt;invoice_key&gt;}</code><br />

tools/conv.py

@@ -1,6 +1,13 @@
 import psycopg2
 import sqlite3
 import os
+import argparse
+
+
+from environs import Env  # type: ignore
+
+env = Env()
+env.read_env()
 
 # Python script to migrate an LNbits SQLite DB to Postgres
 # All credits to @Fritz446 for the awesome work
@@ -11,13 +18,27 @@ import os
 
 # Change these values as needed
 
+
 sqfolder = "data/"
+
+LNBITS_DATABASE_URL = env.str("LNBITS_DATABASE_URL", default=None)
+if LNBITS_DATABASE_URL is None:
     pgdb = "lnbits"
-pguser = "postgres"
+    pguser = "lnbits"
-pgpswd = "yourpassword"
+    pgpswd = "postgres"
     pghost = "localhost"
     pgport = "5432"
     pgschema = ""
+else:
+    # parse postgres://lnbits:postgres@localhost:5432/lnbits
+    pgdb = LNBITS_DATABASE_URL.split("/")[-1]
+    pguser = LNBITS_DATABASE_URL.split("@")[0].split(":")[-2][2:]
+    pgpswd = LNBITS_DATABASE_URL.split("@")[0].split(":")[-1]
+    pghost = LNBITS_DATABASE_URL.split("@")[1].split(":")[0]
+    pgport = LNBITS_DATABASE_URL.split("@")[1].split(":")[1].split("/")[0]
+    pgschema = ""
+
+print(pgdb, pguser, pgpswd, pghost, pgport, pgschema)
 
 
 def get_sqlite_cursor(sqdb) -> sqlite3:
@@ -35,8 +56,6 @@ def get_postgres_cursor():
 def check_db_versions(sqdb):
     sqlite = get_sqlite_cursor(sqdb)
     dblite = dict(sqlite.execute("SELECT * FROM dbversions;").fetchall())
-    if "lnurlpos" in dblite:
-        del dblite["lnurlpos"]
     sqlite.close()
 
     postgres = get_postgres_cursor()
@@ -128,9 +147,14 @@ def migrate_core(sqlite_db_file):
     print("Migrated: core")
 
 
-def migrate_ext(sqlite_db_file, schema):
+def migrate_ext(sqlite_db_file, schema, ignore_missing=True):
-    sq = get_sqlite_cursor(sqlite_db_file)
 
+    # skip this file it has been moved to ext_lnurldevices.sqlite3
+    if sqlite_db_file == "data/ext_lnurlpos.sqlite3":
+        return
+
+    print(f"Migrating {sqlite_db_file}.{schema}")
+    sq = get_sqlite_cursor(sqlite_db_file)
     if schema == "bleskomat":
         # BLESKOMAT LNURLS
         res = sq.execute("SELECT * FROM bleskomat_lnurls;")
@@ -515,19 +539,19 @@ def migrate_ext(sqlite_db_file, schema):
         items = res.fetchall()
         insert_to_pg(q, items)
         fix_id("offlineshop.items_id_seq", items)
-    elif schema == "lnurlpos":
+    elif schema == "lnurlpos" or schema == "lnurldevice":
-        # LNURLPOSS
+        # lnurldevice
-        res = sq.execute("SELECT * FROM lnurlposs;")
+        res = sq.execute("SELECT * FROM lnurldevices;")
         q = f"""
-            INSERT INTO lnurlpos.lnurlposs (id, key, title, wallet, currency, timestamp)
+            INSERT INTO lnurldevice.lnurldevices (id, key, title, wallet, currency, device, profit)
-            VALUES (%s, %s, %s, %s, %s, to_timestamp(%s));
+            VALUES (%s, %s, %s, %s, %s, %s, %s);
         """
         insert_to_pg(q, res.fetchall())
-        # LNURLPOS PAYMENT
+        # lnurldevice PAYMENT
-        res = sq.execute("SELECT * FROM lnurlpospayment;")
+        res = sq.execute("SELECT * FROM lnurldevicepayment;")
         q = f"""
-            INSERT INTO lnurlpos.lnurlpospayment (id, posid, payhash, payload, pin, sats, timestamp)
+            INSERT INTO lnurldevice.lnurldevicepayment (id, deviceid, payhash, payload, pin, sats)
-            VALUES (%s, %s, %s, %s, %s, %s, to_timestamp(%s));
+            VALUES (%s, %s, %s, %s, %s, %s);
         """
         insert_to_pg(q, res.fetchall())
     elif schema == "lnurlp":
@@ -546,9 +570,10 @@ def migrate_ext(sqlite_db_file, schema):
                 success_url,
                 currency,
                 comment_chars,
-                max
+                max,
+                fiat_base_multiplier
             )
-            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
         """
         pay_links = res.fetchall()
         insert_to_pg(q, pay_links)
@@ -637,17 +662,79 @@ def migrate_ext(sqlite_db_file, schema):
         tracks = res.fetchall()
         insert_to_pg(q, tracks)
         fix_id("livestream.tracks_id_seq", tracks)
+    elif schema == "lnaddress":
+        # DOMAINS
+        res = sq.execute("SELECT * FROM domain;")
+        q = f"""
+            INSERT INTO lnaddress.domain(
+	        id, wallet, domain, webhook, cf_token, cf_zone_id, cost, "time")
+	        VALUES (%s, %s, %s, %s, %s, %s, %s, to_timestamp(%s));
+        """
+        insert_to_pg(q, res.fetchall())
+        # ADDRESSES
+        res = sq.execute("SELECT * FROM address;")
+        q = f"""
+            INSERT INTO lnaddress.address(
+            id, wallet, domain, email, username, wallet_key, wallet_endpoint, sats, duration, paid, "time")
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s::boolean, to_timestamp(%s));
+        """
+        insert_to_pg(q, res.fetchall())
+    elif schema == "discordbot":
+        # USERS
+        res = sq.execute("SELECT * FROM users;")
+        q = f"""
+            INSERT INTO discordbot.users(
+	        id, name, admin, discord_id)
+	        VALUES (%s, %s, %s, %s);
+        """
+        insert_to_pg(q, res.fetchall())
+        # WALLETS
+        res = sq.execute("SELECT * FROM wallets;")
+        q = f"""
+            INSERT INTO discordbot.wallets(
+            id, admin, name, "user", adminkey, inkey)
+            VALUES (%s, %s, %s, %s, %s, %s);
+        """
+        insert_to_pg(q, res.fetchall())
     else:
-        print(f"Not implemented: {schema}")
+        print(f"❌ Not implemented: {schema}")
         sq.close()
+
+        if ignore_missing == False:
+            raise Exception(
+                f"Not implemented: {schema}. Use --ignore-missing to skip missing extensions."
+            )
         return
 
-    print(f"Migrated: {schema}")
+    print(f"✅ Migrated: {schema}")
     sq.close()
 
 
-check_db_versions("data/database.sqlite3")
+parser = argparse.ArgumentParser(description="Migrate data from SQLite to PostgreSQL")
-migrate_core("data/database.sqlite3")
+parser.add_argument(
+    dest="sqlite_file",
+    const=True,
+    nargs="?",
+    help="SQLite DB to migrate from",
+    default="data/database.sqlite3",
+    type=str,
+)
+parser.add_argument(
+    "-i",
+    "--dont-ignore-missing",
+    help="Error if migration is missing for an extension.",
+    required=False,
+    default=False,
+    const=True,
+    nargs="?",
+    type=bool,
+)
+args = parser.parse_args()
+
+print(args)
+
+check_db_versions(args.sqlite_file)
+migrate_core(args.sqlite_file)
 
 files = os.listdir(sqfolder)
 for file in files:
@@ -655,4 +742,4 @@ for file in files:
     if file.startswith("ext_"):
         schema = file.replace("ext_", "").split(".")[0]
         print(f"Migrating: {schema}")
-        migrate_ext(path, schema)
+        migrate_ext(path, schema, ignore_missing=not args.dont_ignore_missing)