From c0f66989cba452e81a784c865addc181e27390b3 Mon Sep 17 00:00:00 2001 From: jackstar12 <62219658+jackstar12@users.noreply.github.com> Date: Tue, 9 May 2023 10:18:53 +0200 Subject: [PATCH] Serverside Pagination for payments (#1613) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * initial backend support * implement payments pagination on frontend * implement search for payments api * fix pyright issues * sqlite support for searching * backwards compatability * formatting, small fixes * small optimization * fix sorting issue, add error handling * GET payments test * filter by dates, use List instead of list * fix sqlite * update bundle * test old payments endpoint aswell * refactor for easier review * optimise test * revert unnecessary change --------- Co-authored-by: dni ⚡ --- lnbits/core/crud.py | 84 ++++--- lnbits/core/models.py | 22 +- lnbits/core/static/js/service-worker.js | 2 +- lnbits/core/static/js/wallet.js | 48 +++- lnbits/core/templates/core/wallet.html | 11 +- lnbits/core/views/api.py | 45 +++- lnbits/db.py | 279 ++++++++++++++++++------ lnbits/decorators.py | 23 +- lnbits/helpers.py | 10 +- lnbits/static/bundle.min.js | 2 +- lnbits/static/js/base.js | 11 +- tests/core/views/test_api.py | 65 ++++++ 12 files changed, 460 insertions(+), 142 deletions(-) diff --git a/lnbits/core/crud.py b/lnbits/core/crud.py index c1253802..379cf519 100644 --- a/lnbits/core/crud.py +++ b/lnbits/core/crud.py @@ -7,12 +7,12 @@ from uuid import uuid4 import shortuuid from lnbits import bolt11 -from lnbits.db import COCKROACH, POSTGRES, Connection, Filters +from lnbits.db import Connection, Filters, Page from lnbits.extension_manager import InstallableExtension from lnbits.settings import AdminSettings, EditableSettings, SuperSettings, settings from . import db -from .models import BalanceCheck, Payment, TinyURL, User, Wallet +from .models import BalanceCheck, Payment, PaymentFilters, TinyURL, User, Wallet # accounts # -------- @@ -343,7 +343,7 @@ async def get_latest_payments_by_extension(ext_name: str, ext_id: str, limit: in return rows -async def get_payments( +async def get_payments_paginated( *, wallet_id: Optional[str] = None, complete: bool = False, @@ -352,28 +352,23 @@ async def get_payments( incoming: bool = False, since: Optional[int] = None, exclude_uncheckable: bool = False, - filters: Optional[Filters[Payment]] = None, + filters: Optional[Filters[PaymentFilters]] = None, conn: Optional[Connection] = None, -) -> List[Payment]: +) -> Page[Payment]: """ Filters payments to be returned by complete | pending | outgoing | incoming. """ - args: List[Any] = [] + values: List[Any] = [] clause: List[str] = [] if since is not None: - if db.type == POSTGRES: - clause.append("time > to_timestamp(?)") - elif db.type == COCKROACH: - clause.append("time > cast(? AS timestamp)") - else: - clause.append("time > ?") - args.append(since) + clause.append(f"time > {db.timestamp_placeholder}") + values.append(since) if wallet_id: clause.append("wallet = ?") - args.append(wallet_id) + values.append(wallet_id) if complete and pending: pass @@ -397,21 +392,54 @@ async def get_payments( clause.append("checking_id NOT LIKE 'temp_%'") clause.append("checking_id NOT LIKE 'internal_%'") - if not filters: - filters = Filters(limit=None, offset=None) - - rows = await (conn or db).fetchall( - f""" - SELECT * - FROM apipayments - {filters.where(clause)} - ORDER BY time DESC - {filters.pagination()} - """, - filters.values(args), + return await (conn or db).fetch_page( + "SELECT * FROM apipayments", + clause, + values, + filters=filters, + model=Payment, ) - return [Payment.from_row(row) for row in rows] + +async def get_payments( + *, + wallet_id: Optional[str] = None, + complete: bool = False, + pending: bool = False, + outgoing: bool = False, + incoming: bool = False, + since: Optional[int] = None, + exclude_uncheckable: bool = False, + filters: Optional[Filters[PaymentFilters]] = None, + conn: Optional[Connection] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, +) -> list[Payment]: + """ + Filters payments to be returned by complete | pending | outgoing | incoming. + """ + + if not filters: + filters = Filters() + + if limit: + filters.limit = limit + if offset: + filters.offset = offset + + page = await get_payments_paginated( + wallet_id=wallet_id, + complete=complete, + pending=pending, + outgoing=outgoing, + incoming=incoming, + since=since, + exclude_uncheckable=exclude_uncheckable, + filters=filters, + conn=conn, + ) + + return page.data async def delete_expired_invoices( @@ -454,7 +482,6 @@ async def create_payment( webhook: Optional[str] = None, conn: Optional[Connection] = None, ) -> Payment: - # todo: add this when tests are fixed # previous_payment = await get_wallet_payment(wallet_id, payment_hash, conn=conn) # assert previous_payment is None, "Payment already exists" @@ -514,7 +541,6 @@ async def update_payment_details( new_checking_id: Optional[str] = None, conn: Optional[Connection] = None, ) -> None: - set_clause: List[str] = [] set_variables: List[Any] = [] diff --git a/lnbits/core/models.py b/lnbits/core/models.py index 4bcdd331..dae4a1e0 100644 --- a/lnbits/core/models.py +++ b/lnbits/core/models.py @@ -11,7 +11,7 @@ from lnurl import encode as lnurl_encode from loguru import logger from pydantic import BaseModel -from lnbits.db import Connection +from lnbits.db import Connection, FilterModel, FromRowModel from lnbits.helpers import url_for from lnbits.settings import get_wallet_class, settings from lnbits.wallets.base import PaymentStatus @@ -86,7 +86,7 @@ class User(BaseModel): return False -class Payment(BaseModel): +class Payment(FromRowModel): checking_id: str pending: bool amount: int @@ -214,6 +214,24 @@ class Payment(BaseModel): await delete_payment(self.checking_id, conn=conn) +class PaymentFilters(FilterModel): + __search_fields__ = ["memo", "amount"] + + checking_id: str + amount: int + fee: int + memo: Optional[str] + time: datetime.datetime + bolt11: str + preimage: str + payment_hash: str + expiry: Optional[datetime.datetime] + extra: Dict = {} + wallet_id: str + webhook: Optional[str] + webhook_status: Optional[int] + + class BalanceCheck(BaseModel): wallet: str service: str diff --git a/lnbits/core/static/js/service-worker.js b/lnbits/core/static/js/service-worker.js index 9684739f..cf89ca89 100644 --- a/lnbits/core/static/js/service-worker.js +++ b/lnbits/core/static/js/service-worker.js @@ -1,6 +1,6 @@ // update cache version every time there is a new deployment // so the service worker reinitializes the cache -const CACHE_VERSION = 5 +const CACHE_VERSION = 6 const CURRENT_CACHE = `lnbits-${CACHE_VERSION}-` const getApiKey = request => { diff --git a/lnbits/core/static/js/wallet.js b/lnbits/core/static/js/wallet.js index e2708ad6..585b1dc7 100644 --- a/lnbits/core/static/js/wallet.js +++ b/lnbits/core/static/js/wallet.js @@ -152,14 +152,14 @@ new Vue({ field: 'memo' }, { - name: 'date', + name: 'time', align: 'left', label: this.$t('date'), field: 'date', sortable: true }, { - name: 'sat', + name: 'amount', align: 'right', label: this.$t('amount') + ' (' + LNBITS_DENOMINATION + ')', field: 'sat', @@ -173,9 +173,14 @@ new Vue({ } ], pagination: { - rowsPerPage: 10 + rowsPerPage: 10, + page: 1, + sortBy: 'time', + descending: true, + rowsNumber: 10 }, - filter: null + filter: null, + loading: false }, paymentsChart: { show: false @@ -695,16 +700,35 @@ new Vue({ LNbits.href.deleteWallet(walletId, user) }) }, - fetchPayments: function () { - return LNbits.api.getPayments(this.g.wallet).then(response => { - this.payments = response.data - .map(obj => { + fetchPayments: function (props) { + // Props are passed by qasar when pagination or sorting changes + if (props) { + this.paymentsTable.pagination = props.pagination + } + let pagination = this.paymentsTable.pagination + this.paymentsTable.loading = true + const query = { + limit: pagination.rowsPerPage, + offset: (pagination.page - 1) * pagination.rowsPerPage, + sortby: pagination.sortBy ?? 'time', + direction: pagination.descending ? 'desc' : 'asc' + } + if (this.paymentsTable.filter) { + query.search = this.paymentsTable.filter + } + return LNbits.api + .getPayments(this.g.wallet, query) + .then(response => { + this.paymentsTable.loading = false + this.paymentsTable.pagination.rowsNumber = response.data.total + this.payments = response.data.data.map(obj => { return LNbits.map.payment(obj) }) - .sort((a, b) => { - return b.time - a.time - }) - }) + }) + .catch(err => { + this.paymentsTable.loading = false + LNbits.utils.notifyApiError(err) + }) }, fetchBalance: function () { LNbits.api.getWallet(this.g.wallet).then(response => { diff --git a/lnbits/core/templates/core/wallet.html b/lnbits/core/templates/core/wallet.html index 00f36b23..896304e2 100644 --- a/lnbits/core/templates/core/wallet.html +++ b/lnbits/core/templates/core/wallet.html @@ -125,7 +125,6 @@ {% raw %}