fix migrations, add documentation, hopefully fix workflow (#899)

* fix migrations, add documentation, hopefully fix workflow

* renaming to test-migration and add migration

Co-authored-by: dni <dni.khr@gmail.com>

.github/workflows/migrations.yml

@@ -9,9 +9,9 @@ jobs:
       postgres:
         image: postgres:latest
         env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
+          POSTGRES_USER: lnbits
+          POSTGRES_PASSWORD: lnbits
+          POSTGRES_DB: migration
         ports:
           # maps tcp port 5432 on service container to the host
           - 5432:5432
@@ -36,11 +36,4 @@ jobs:
           sudo apt install unzip
       - name: Run migrations
         run: |
-          rm -rf ./data
-          mkdir -p ./data
-          export LNBITS_DATA_FOLDER="./data"
-          unzip tests/data/mock_data.zip -d ./data
-          timeout 5s poetry run lnbits --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
-          export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres"
-          timeout 5s poetry run lnbits --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
-          poetry run python tools/conv.py
+          make test-migration

Makefile

@@ -4,7 +4,7 @@ all: format check requirements.txt
 
 format: prettier isort black
 
-check: mypy checkprettier checkisort checkblack 
+check: mypy checkprettier checkisort checkblack
 
 prettier: $(shell find lnbits -name "*.js" -name ".html")
 	./node_modules/.bin/prettier --write lnbits/static/js/*.js lnbits/core/static/js/*.js lnbits/extensions/*/templates/*/*.html ./lnbits/core/templates/core/*.html lnbits/templates/*.html lnbits/extensions/*/static/js/*.js lnbits/extensions/*/static/components/*/*.js  lnbits/extensions/*/static/components/*/*.html
@@ -46,5 +46,24 @@ test-venv:
 	PYTHONUNBUFFERED=1 \
 	./venv/bin/pytest --durations=1 -s --cov=lnbits --cov-report=xml tests
 
+test-migration:
+	rm -rf ./migration-data
+	mkdir -p ./migration-data
+	unzip tests/data/mock_data.zip -d ./migration-data
+	HOST=0.0.0.0 \
+	PORT=5002 \
+	LNBITS_DATA_FOLDER="./migration-data" \
+	timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+	HOST=0.0.0.0 \
+	PORT=5002 \
+	LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \
+	timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+	LNBITS_DATA_FOLDER="./migration-data" \
+	LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \
+	poetry run python tools/conv.py
+
+migration:
+	poetry run python tools/conv.py
+
 bak:
 	# LNBITS_DATABASE_URL=postgres://postgres:postgres@0.0.0.0:5432/postgres

docs/devs/extensions.md

@@ -48,4 +48,25 @@ LNbits currently supports SQLite and PostgreSQL databases. There is a migration
 
 ### Adding mock data to `mock_data.zip`
 
-`mock_data.zip` contains a few lines of sample SQLite data and is used in automated GitHub test to see whether your migration in `conv.py` works. Run your extension and save a few lines of data into a SQLite `your_extension.sqlite3` file. Unzip `tests/data/mock_data.zip`, add `your_extension.sqlite3` and zip it again. Add the updated `mock_data.zip` to your PR.
+`mock_data.zip` contains a few lines of sample SQLite data and is used in automated GitHub test to see whether your migration in `conv.py` works. Run your extension and save a few lines of data into a SQLite `your_extension.sqlite3` file. Unzip `tests/data/mock_data.zip`, add `your_extension.sqlite3`, updated `database.sqlite3` and zip it again. Add the updated `mock_data.zip` to your PR.
+
+### running migration locally
+you will need a running postgres database
+
+#### create lnbits user for migration database
+```console
+sudo su - postgres -c "psql -c 'CREATE ROLE lnbits LOGIN PASSWORD 'lnbits';'"
+```
+#### create migration database
+```console
+sudo su - postgres -c "psql -c 'CREATE DATABASE migration;'"
+```
+#### run the migration
+```console
+make test-migration
+```
+sudo su - postgres -c "psql -c 'CREATE ROLE lnbits LOGIN PASSWORD 'lnbits';'"
+#### clean migration database afterwards, fails if you try again
+```console
+sudo su - postgres -c "psql -c 'DROP DATABASE IF EXISTS migration;'"
+```

docs/guide/installation.md

@@ -170,8 +170,9 @@ LNBITS_DATABASE_URL="postgres://postgres:postgres@localhost/lnbits"
 
 # START LNbits
 # STOP LNbits
-# on the LNBits folder, locate and edit 'tools/conv.py' with the relevant credentials
-python3 tools/conv.py
+poetry run python tools/conv.py
+# or
+make migration
 ```
 
 Hopefully, everything works and get migrated... Launch LNbits again and check if everything is working properly.

tools/conv.py

@@ -19,16 +19,12 @@ env.read_env()
 # Change these values as needed
 
 
-sqfolder = "data/"
+sqfolder = env.str("LNBITS_DATA_FOLDER", default=None)
 
 LNBITS_DATABASE_URL = env.str("LNBITS_DATABASE_URL", default=None)
 if LNBITS_DATABASE_URL is None:
-    pgdb = "lnbits"
-    pguser = "lnbits"
-    pgpswd = "postgres"
-    pghost = "localhost"
-    pgport = "5432"
-    pgschema = ""
+    print("missing LNBITS_DATABASE_URL")
+    sys.exit(1)
 else:
     # parse postgres://lnbits:postgres@localhost:5432/lnbits
     pgdb = LNBITS_DATABASE_URL.split("/")[-1]
@@ -129,7 +125,7 @@ def migrate_db(file: str, schema: str, exclude_tables: List[str] = []):
     sq = get_sqlite_cursor(file)
     tables = sq.execute(
         """
-        SELECT name FROM sqlite_master 
+        SELECT name FROM sqlite_master
         WHERE type='table' AND name not like 'sqlite?_%' escape '?'
     """
     ).fetchall()