heierbtc/lnbits
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CI: DB migration with real data (#776)

Browse Source 
* fix migration conv.py

* migrate with mock data
This commit is contained in:
calle 2022-07-25 09:13:41 +02:00 committed by GitHub
parent 4b5e113c03
commit 4bb184e08b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 74 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/migrations.yml vendored
View File

@ -38,12 +38,14 @@ jobs:
./venv/bin/python -m pip install --upgrade pip
./venv/bin/pip install -r requirements.txt
./venv/bin/pip install pytest pytest-asyncio pytest-cov requests mock
sudo apt install unzip
- name: Run migrations
run: |
rm -rf ./data
mkdir -p ./data
export LNBITS_DATA_FOLDER="./data"
unzip tests/data/mock_data.zip -d ./data
timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres"
timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
./venv/bin/python tools/conv.py --dont-ignore-missing
./venv/bin/python tools/conv.py

2
.gitignore vendored
View File

@ -15,7 +15,7 @@ __pycache__
.webassets-cache
htmlcov
test-reports
tests/data
tests/data/*.sqlite3
*.swo
*.swp

BIN
tests/data/mock_data.zip Normal file
View File

Binary file not shown.

103
tools/conv.py
View File

@ -38,8 +38,6 @@ else:
pgport = LNBITS_DATABASE_URL.split("@")[1].split(":")[1].split("/")[0]
pgschema = ""
print(pgdb, pguser, pgpswd, pghost, pgport, pgschema)
def get_sqlite_cursor(sqdb) -> sqlite3:
consq = sqlite3.connect(sqdb)
@ -99,8 +97,12 @@ def insert_to_pg(query, data):
for d in data:
try:
cursor.execute(query, d)
except:
raise ValueError(f"Failed to insert {d}")
except Exception as e:
if args.ignore_errors:
print(e)
print(f"Failed to insert {d}")
else:
raise ValueError(f"Failed to insert {d}")
connection.commit()
cursor.close()
@ -256,9 +258,10 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True):
k1,
open_time,
used,
usescsv
usescsv,
webhook_url
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
"""
insert_to_pg(q, res.fetchall())
# WITHDRAW HASH CHECK
@ -316,8 +319,8 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True):
# TPOSS
res = sq.execute("SELECT * FROM tposs;")
q = f"""
INSERT INTO tpos.tposs (id, wallet, name, currency)
VALUES (%s, %s, %s, %s);
INSERT INTO tpos.tposs (id, wallet, name, currency, tip_wallet, tip_options)
VALUES (%s, %s, %s, %s, %s, %s);
"""
insert_to_pg(q, res.fetchall())
elif schema == "tipjar":
@ -512,12 +515,13 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True):
wallet,
url,
memo,
description,
amount,
time,
remembers,
extra
extras
)
VALUES (%s, %s, %s, %s, %s, to_timestamp(%s), %s, %s);
VALUES (%s, %s, %s, %s, %s, %s, to_timestamp(%s), %s, %s);
"""
insert_to_pg(q, res.fetchall())
elif schema == "offlineshop":
@ -543,15 +547,15 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True):
# lnurldevice
res = sq.execute("SELECT * FROM lnurldevices;")
q = f"""
INSERT INTO lnurldevice.lnurldevices (id, key, title, wallet, currency, device, profit)
VALUES (%s, %s, %s, %s, %s, %s, %s);
INSERT INTO lnurldevice.lnurldevices (id, key, title, wallet, currency, device, profit, timestamp)
VALUES (%s, %s, %s, %s, %s, %s, %s, to_timestamp(%s));
"""
insert_to_pg(q, res.fetchall())
# lnurldevice PAYMENT
res = sq.execute("SELECT * FROM lnurldevicepayment;")
q = f"""
INSERT INTO lnurldevice.lnurldevicepayment (id, deviceid, payhash, payload, pin, sats)
VALUES (%s, %s, %s, %s, %s, %s);
INSERT INTO lnurldevice.lnurldevicepayment (id, deviceid, payhash, payload, pin, sats, timestamp)
VALUES (%s, %s, %s, %s, %s, %s, to_timestamp(%s));
"""
insert_to_pg(q, res.fetchall())
elif schema == "lnurlp":
@ -710,36 +714,69 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True):
sq.close()
parser = argparse.ArgumentParser(description="Migrate data from SQLite to PostgreSQL")
parser = argparse.ArgumentParser(
description="LNbits migration tool for migrating data from SQLite to PostgreSQL"
)
parser.add_argument(
dest="sqlite_file",
dest="sqlite_path",
const=True,
nargs="?",
help="SQLite DB to migrate from",
default="data/database.sqlite3",
help=f"SQLite DB folder *or* single extension db file to migrate. Default: {sqfolder}",
default=sqfolder,
type=str,
)
parser.add_argument(
"-i",
"--dont-ignore-missing",
help="Error if migration is missing for an extension.",
"-e",
"--extensions-only",
help="Migrate only extensions",
required=False,
default=False,
const=True,
nargs="?",
type=bool,
action="store_true",
)
parser.add_argument(
"-s",
"--skip-missing",
help="Error if migration is missing for an extension",
required=False,
default=False,
action="store_true",
)
parser.add_argument(
"-i",
"--ignore-errors",
help="Don't error if migration fails",
required=False,
default=False,
action="store_true",
)
args = parser.parse_args()
print(args)
print("Selected path: ", args.sqlite_path)
check_db_versions(args.sqlite_file)
migrate_core(args.sqlite_file)
if os.path.isdir(args.sqlite_path):
file = os.path.join(args.sqlite_path, "database.sqlite3")
check_db_versions(file)
if not args.extensions_only:
print(f"Migrating: {file}")
migrate_core(file)
if os.path.isdir(args.sqlite_path):
files = [
os.path.join(args.sqlite_path, file) for file in os.listdir(args.sqlite_path)
]
else:
files = [args.sqlite_path]
files = os.listdir(sqfolder)
for file in files:
path = f"data/{file}"
if file.startswith("ext_"):
schema = file.replace("ext_", "").split(".")[0]
print(f"Migrating: {schema}")
migrate_ext(path, schema, ignore_missing=not args.dont_ignore_missing)
filename = os.path.basename(file)
if filename.startswith("ext_"):
schema = filename.replace("ext_", "").split(".")[0]
print(f"Migrating: {file}")
migrate_ext(
file,
schema,
ignore_missing=args.skip_missing,
)