CI: Migration SQLite to PostgreSQL (#719)

* migrate all extensions * errors if migration is missing
2022-07-17 13:11:13 +02:00 · 2022-07-17 13:11:13 +02:00 · ad2aad05e0
commit ad2aad05e0
parent f4e7d62ca3
6 changed files with 167 additions and 31 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -4,6 +4,7 @@ docker
 docs
 tests
 venv
+tools

 *.md
 *.log
@ -12,7 +13,6 @@ venv

 .gitignore
 .prettierrc
-conv.py
 LICENSE
 Makefile
 mypy.ini
--- a/.github/workflows/migrations.yml
+++ b/.github/workflows/migrations.yml
@ -0,0 +1,49 @@
+name: migrations
+
+on: [pull_request]
+
+jobs:
+  sqlite-to-postgres:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          # maps tcp port 5432 on service container to the host
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      matrix:
+        python-version: [3.8]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        env: 
+          VIRTUAL_ENV: ./venv
+          PATH: ${{ env.VIRTUAL_ENV }}/bin:${{ env.PATH }}
+        run: |
+          python -m venv ${{ env.VIRTUAL_ENV }}
+          ./venv/bin/python -m pip install --upgrade pip
+          ./venv/bin/pip install -r requirements.txt
+          ./venv/bin/pip install pytest pytest-asyncio pytest-cov requests mock
+      - name: Run migrations
+        run: |
+          rm -rf ./data
+          mkdir -p ./data
+          export LNBITS_DATA_FOLDER="./data"
+          timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+          export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres"
+          timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+          ./venv/bin/python tools/conv.py --dont-ignore-missing
--- a/docs/guide/installation.md
+++ b/docs/guide/installation.md
@ -142,8 +142,8 @@ LNBITS_DATABASE_URL="postgres://postgres:postgres@localhost/lnbits"

 # START LNbits
 # STOP LNbits
-# on the LNBits folder, locate and edit 'conv.py' with the relevant credentials
-python3 conv.py
+# on the LNBits folder, locate and edit 'tools/conv.py' with the relevant credentials
+python3 tools/conv.py
 ```

 Hopefully, everything works and get migrated... Launch LNbits again and check if everything is working properly.
--- a/lnbits/extensions/lnurldevice/migrations.py
+++ b/lnbits/extensions/lnurldevice/migrations.py
@ -42,7 +42,7 @@ async def m002_redux(db):
    """
    try:
        for row in [
-            list(row) for row in await db2.fetchall("SELECT * FROM lnurlpos.lnurlposs")
+            list(row) for row in await db2.fetchall("SELECT * FROM lnurlpos.lnurlpos")
        ]:
            await db.execute(
                """
--- a/lnbits/extensions/lnurldevice/templates/lnurldevice/_api_docs.html
+++ b/lnbits/extensions/lnurldevice/templates/lnurldevice/_api_docs.html
@ -120,7 +120,7 @@
        <q-card-section>
          <code
            ><span class="text-blue">GET</span>
-            /lnurldevice/api/v1/lnurlposs</code
+            /lnurldevice/api/v1/lnurlpos</code
          >
          <h5 class="text-caption q-mt-sm q-mb-none">Headers</h5>
          <code>{"X-Api-Key": &lt;invoice_key&gt;}</code><br />
--- a/tools/conv.py
+++ b/tools/conv.py
@ -1,6 +1,13 @@
 import psycopg2
 import sqlite3
 import os
+import argparse
+
+
+from environs import Env  # type: ignore
+
+env = Env()
+env.read_env()

 # Python script to migrate an LNbits SQLite DB to Postgres
 # All credits to @Fritz446 for the awesome work
@ -11,13 +18,27 @@ import os

 # Change these values as needed

+
 sqfolder = "data/"
+
+LNBITS_DATABASE_URL = env.str("LNBITS_DATABASE_URL", default=None)
+if LNBITS_DATABASE_URL is None:
    pgdb = "lnbits"
-pguser = "postgres"
-pgpswd = "yourpassword"
+    pguser = "lnbits"
+    pgpswd = "postgres"
    pghost = "localhost"
    pgport = "5432"
    pgschema = ""
+else:
+    # parse postgres://lnbits:postgres@localhost:5432/lnbits
+    pgdb = LNBITS_DATABASE_URL.split("/")[-1]
+    pguser = LNBITS_DATABASE_URL.split("@")[0].split(":")[-2][2:]
+    pgpswd = LNBITS_DATABASE_URL.split("@")[0].split(":")[-1]
+    pghost = LNBITS_DATABASE_URL.split("@")[1].split(":")[0]
+    pgport = LNBITS_DATABASE_URL.split("@")[1].split(":")[1].split("/")[0]
+    pgschema = ""
+
+print(pgdb, pguser, pgpswd, pghost, pgport, pgschema)


 def get_sqlite_cursor(sqdb) -> sqlite3:
@ -35,8 +56,6 @@ def get_postgres_cursor():
 def check_db_versions(sqdb):
    sqlite = get_sqlite_cursor(sqdb)
    dblite = dict(sqlite.execute("SELECT * FROM dbversions;").fetchall())
-    if "lnurlpos" in dblite:
-        del dblite["lnurlpos"]
    sqlite.close()

    postgres = get_postgres_cursor()
@ -128,9 +147,14 @@ def migrate_core(sqlite_db_file):
    print("Migrated: core")


-def migrate_ext(sqlite_db_file, schema):
-    sq = get_sqlite_cursor(sqlite_db_file)
+def migrate_ext(sqlite_db_file, schema, ignore_missing=True):

+    # skip this file it has been moved to ext_lnurldevices.sqlite3
+    if sqlite_db_file == "data/ext_lnurlpos.sqlite3":
+        return
+
+    print(f"Migrating {sqlite_db_file}.{schema}")
+    sq = get_sqlite_cursor(sqlite_db_file)
    if schema == "bleskomat":
        # BLESKOMAT LNURLS
        res = sq.execute("SELECT * FROM bleskomat_lnurls;")
@ -515,19 +539,19 @@ def migrate_ext(sqlite_db_file, schema):
        items = res.fetchall()
        insert_to_pg(q, items)
        fix_id("offlineshop.items_id_seq", items)
-    elif schema == "lnurlpos":
-        # LNURLPOSS
-        res = sq.execute("SELECT * FROM lnurlposs;")
+    elif schema == "lnurlpos" or schema == "lnurldevice":
+        # lnurldevice
+        res = sq.execute("SELECT * FROM lnurldevices;")
        q = f"""
-            INSERT INTO lnurlpos.lnurlposs (id, key, title, wallet, currency, timestamp)
-            VALUES (%s, %s, %s, %s, %s, to_timestamp(%s));
+            INSERT INTO lnurldevice.lnurldevices (id, key, title, wallet, currency, device, profit)
+            VALUES (%s, %s, %s, %s, %s, %s, %s);
        """
        insert_to_pg(q, res.fetchall())
-        # LNURLPOS PAYMENT
-        res = sq.execute("SELECT * FROM lnurlpospayment;")
+        # lnurldevice PAYMENT
+        res = sq.execute("SELECT * FROM lnurldevicepayment;")
        q = f"""
-            INSERT INTO lnurlpos.lnurlpospayment (id, posid, payhash, payload, pin, sats, timestamp)
-            VALUES (%s, %s, %s, %s, %s, %s, to_timestamp(%s));
+            INSERT INTO lnurldevice.lnurldevicepayment (id, deviceid, payhash, payload, pin, sats)
+            VALUES (%s, %s, %s, %s, %s, %s);
        """
        insert_to_pg(q, res.fetchall())
    elif schema == "lnurlp":
@ -546,9 +570,10 @@ def migrate_ext(sqlite_db_file, schema):
                success_url,
                currency,
                comment_chars,
-                max
+                max,
+                fiat_base_multiplier
            )
-            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
        """
        pay_links = res.fetchall()
        insert_to_pg(q, pay_links)
@ -637,17 +662,79 @@ def migrate_ext(sqlite_db_file, schema):
        tracks = res.fetchall()
        insert_to_pg(q, tracks)
        fix_id("livestream.tracks_id_seq", tracks)
+    elif schema == "lnaddress":
+        # DOMAINS
+        res = sq.execute("SELECT * FROM domain;")
+        q = f"""
+            INSERT INTO lnaddress.domain(
+	        id, wallet, domain, webhook, cf_token, cf_zone_id, cost, "time")
+	        VALUES (%s, %s, %s, %s, %s, %s, %s, to_timestamp(%s));
+        """
+        insert_to_pg(q, res.fetchall())
+        # ADDRESSES
+        res = sq.execute("SELECT * FROM address;")
+        q = f"""
+            INSERT INTO lnaddress.address(
+            id, wallet, domain, email, username, wallet_key, wallet_endpoint, sats, duration, paid, "time")
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s::boolean, to_timestamp(%s));
+        """
+        insert_to_pg(q, res.fetchall())
+    elif schema == "discordbot":
+        # USERS
+        res = sq.execute("SELECT * FROM users;")
+        q = f"""
+            INSERT INTO discordbot.users(
+	        id, name, admin, discord_id)
+	        VALUES (%s, %s, %s, %s);
+        """
+        insert_to_pg(q, res.fetchall())
+        # WALLETS
+        res = sq.execute("SELECT * FROM wallets;")
+        q = f"""
+            INSERT INTO discordbot.wallets(
+            id, admin, name, "user", adminkey, inkey)
+            VALUES (%s, %s, %s, %s, %s, %s);
+        """
+        insert_to_pg(q, res.fetchall())
    else:
-        print(f"Not implemented: {schema}")
+        print(f"❌ Not implemented: {schema}")
        sq.close()
+
+        if ignore_missing == False:
+            raise Exception(
+                f"Not implemented: {schema}. Use --ignore-missing to skip missing extensions."
+            )
        return

-    print(f"Migrated: {schema}")
+    print(f"✅ Migrated: {schema}")
    sq.close()


-check_db_versions("data/database.sqlite3")
-migrate_core("data/database.sqlite3")
+parser = argparse.ArgumentParser(description="Migrate data from SQLite to PostgreSQL")
+parser.add_argument(
+    dest="sqlite_file",
+    const=True,
+    nargs="?",
+    help="SQLite DB to migrate from",
+    default="data/database.sqlite3",
+    type=str,
+)
+parser.add_argument(
+    "-i",
+    "--dont-ignore-missing",
+    help="Error if migration is missing for an extension.",
+    required=False,
+    default=False,
+    const=True,
+    nargs="?",
+    type=bool,
+)
+args = parser.parse_args()
+
+print(args)
+
+check_db_versions(args.sqlite_file)
+migrate_core(args.sqlite_file)

 files = os.listdir(sqfolder)
 for file in files:
@ -655,4 +742,4 @@ for file in files:
    if file.startswith("ext_"):
        schema = file.replace("ext_", "").split(".")[0]
        print(f"Migrating: {schema}")
-        migrate_ext(path, schema)
+        migrate_ext(path, schema, ignore_missing=not args.dont_ignore_missing)