From 64e78d03a117fffeb18e18154d9028a2532285a5 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Fri, 1 Sep 2023 10:52:15 +0200 Subject: [PATCH] wallet-core: implement and test stored backups --- packages/idb-bridge/src/SqliteBackend.ts | 27 ++++- packages/idb-bridge/src/bridge-idb.ts | 7 +- .../integrationtests/test-stored-backups.ts | 110 ++++++++++++++++++ .../src/integrationtests/testrunner.ts | 2 + packages/taler-util/src/backup-types.ts | 19 +++ packages/taler-util/src/wallet-types.ts | 12 ++ packages/taler-wallet-cli/src/index.ts | 42 ++++++- packages/taler-wallet-core/src/db.ts | 53 ++++++--- .../taler-wallet-core/src/host-impl.node.ts | 6 +- packages/taler-wallet-core/src/wallet.ts | 68 ++++++++++- 10 files changed, 318 insertions(+), 28 deletions(-) create mode 100644 packages/taler-harness/src/integrationtests/test-stored-backups.ts diff --git a/packages/idb-bridge/src/SqliteBackend.ts b/packages/idb-bridge/src/SqliteBackend.ts index c40281861..a25ec0045 100644 --- a/packages/idb-bridge/src/SqliteBackend.ts +++ b/packages/idb-bridge/src/SqliteBackend.ts @@ -1882,7 +1882,7 @@ export class SqliteBackend implements Backend { } } - clearObjectStore( + async clearObjectStore( btx: DatabaseTransaction, objectStoreName: string, ): Promise { @@ -1906,7 +1906,21 @@ export class SqliteBackend implements Backend { ); } - throw new Error("Method not implemented."); + this._prep(sqlClearObjectStore).run({ + object_store_id: scopeInfo.objectStoreId, + }); + + for (const index of scopeInfo.indexMap.values()) { + let stmt: Sqlite3Statement; + if (index.unique) { + stmt = this._prep(sqlClearUniqueIndexData); + } else { + stmt = this._prep(sqlClearIndexData); + } + stmt.run({ + index_id: index.indexId, + }); + } } } @@ -1963,6 +1977,15 @@ CREATE TABLE IF NOT EXISTS unique_index_data ); `; +const sqlClearObjectStore = ` +DELETE FROM object_data WHERE object_store_id=$object_store_id`; + +const sqlClearIndexData = ` +DELETE FROM index_data WHERE index_id=$index_id`; + +const sqlClearUniqueIndexData = ` +DELETE FROM unique_index_data WHERE index_id=$index_id`; + const sqlListDatabases = ` SELECT name, version FROM databases; `; diff --git a/packages/idb-bridge/src/bridge-idb.ts b/packages/idb-bridge/src/bridge-idb.ts index 8cecba534..f3749c77c 100644 --- a/packages/idb-bridge/src/bridge-idb.ts +++ b/packages/idb-bridge/src/bridge-idb.ts @@ -735,7 +735,9 @@ export class BridgeIDBDatabase extends FakeEventTarget implements IDBDatabase { } if (this._closePending) { - throw new InvalidStateError(); + throw new InvalidStateError( + `tried to start transaction on ${this._name}, but a close is pending`, + ); } if (!Array.isArray(storeNames)) { @@ -930,6 +932,9 @@ export class BridgeIDBFactory { // http://www.w3.org/TR/2015/REC-IndexedDB-20150108/#dfn-steps-for-running-a-versionchange-transaction for (const otherConn of this.connections) { + if (otherConn._name != db._name) { + continue; + } if (otherConn._closePending) { continue; } diff --git a/packages/taler-harness/src/integrationtests/test-stored-backups.ts b/packages/taler-harness/src/integrationtests/test-stored-backups.ts new file mode 100644 index 000000000..831506d83 --- /dev/null +++ b/packages/taler-harness/src/integrationtests/test-stored-backups.ts @@ -0,0 +1,110 @@ +/* + This file is part of GNU Taler + (C) 2023 Taler Systems S.A. + + GNU Taler is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNU Taler; see the file COPYING. If not, see + */ + +/** + * Imports. + */ +import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; +import { GlobalTestState } from "../harness/harness.js"; +import { + withdrawViaBankV2, + makeTestPaymentV2, + useSharedTestkudosEnvironment, +} from "../harness/helpers.js"; + +/** + * Test stored backup wallet-core API. + */ +export async function runStoredBackupsTest(t: GlobalTestState) { + // Set up test environment + + const { walletClient, bank, exchange, merchant } = + await useSharedTestkudosEnvironment(t); + + // Withdraw digital cash into the wallet. + + await withdrawViaBankV2(t, { + walletClient, + bank, + exchange, + amount: "TESTKUDOS:20", + }); + + await walletClient.call(WalletApiOperation.TestingWaitTransactionsFinal, {}); + + const sb1Resp = await walletClient.call( + WalletApiOperation.CreateStoredBackup, + {}, + ); + const sbList = await walletClient.call( + WalletApiOperation.ListStoredBackups, + {}, + ); + t.assertTrue(sbList.storedBackups.length === 1); + t.assertTrue(sbList.storedBackups[0].name === sb1Resp.name); + + const order = { + summary: "Buy me!", + amount: "TESTKUDOS:5", + fulfillment_url: "taler://fulfillment-success/thx", + }; + + await makeTestPaymentV2(t, { walletClient, merchant, order }); + await walletClient.call(WalletApiOperation.TestingWaitTransactionsFinal, {}); + + const txn1 = await walletClient.call(WalletApiOperation.GetTransactions, {}); + t.assertDeepEqual(txn1.transactions.length, 2); + + // Recover from the stored backup now. + + const sb2Resp = await walletClient.call( + WalletApiOperation.CreateStoredBackup, + {}, + ); + + console.log("recovering backup"); + + await walletClient.call(WalletApiOperation.RecoverStoredBackup, { + name: sb1Resp.name, + }); + + console.log("first recovery done"); + + // Recovery went well, now we can delete the backup + // of the old database we stored before importing. + { + const sbl1 = await walletClient.call( + WalletApiOperation.ListStoredBackups, + {}, + ); + t.assertTrue(sbl1.storedBackups.length === 2); + + await walletClient.call(WalletApiOperation.DeleteStoredBackup, { + name: sb1Resp.name, + }); + const sbl2 = await walletClient.call( + WalletApiOperation.ListStoredBackups, + {}, + ); + t.assertTrue(sbl2.storedBackups.length === 1); + } + + const txn2 = await walletClient.call(WalletApiOperation.GetTransactions, {}); + // We only have the withdrawal after restoring + t.assertDeepEqual(txn2.transactions.length, 1); +} + +runStoredBackupsTest.suites = ["wallet"]; diff --git a/packages/taler-harness/src/integrationtests/testrunner.ts b/packages/taler-harness/src/integrationtests/testrunner.ts index 501af98a4..7afd9bc83 100644 --- a/packages/taler-harness/src/integrationtests/testrunner.ts +++ b/packages/taler-harness/src/integrationtests/testrunner.ts @@ -114,6 +114,7 @@ import { runSimplePaymentTest } from "./test-simple-payment.js"; import { runTermOfServiceFormatTest } from "./test-tos-format.js"; import { runExchangePurseTest } from "./test-exchange-purse.js"; import { getSharedTestDir } from "../harness/helpers.js"; +import { runStoredBackupsTest } from "./test-stored-backups.js"; /** * Test runner. @@ -212,6 +213,7 @@ const allTests: TestMainFunction[] = [ runWithdrawalFeesTest, runWithdrawalHugeTest, runTermOfServiceFormatTest, + runStoredBackupsTest, ]; export interface TestRunSpec { diff --git a/packages/taler-util/src/backup-types.ts b/packages/taler-util/src/backup-types.ts index 2eba1e4ca..8c38b70a6 100644 --- a/packages/taler-util/src/backup-types.ts +++ b/packages/taler-util/src/backup-types.ts @@ -14,6 +14,8 @@ GNU Taler; see the file COPYING. If not, see */ +import { AmountString } from "./taler-types.js"; + export interface BackupRecovery { walletRootPriv: string; providers: { @@ -21,3 +23,20 @@ export interface BackupRecovery { url: string; }[]; } + +export class BackupBackupProviderTerms { + /** + * Last known supported protocol version. + */ + supported_protocol_version: string; + + /** + * Last known annual fee. + */ + annual_fee: AmountString; + + /** + * Last known storage limit. + */ + storage_limit_in_megabytes: number; +} diff --git a/packages/taler-util/src/wallet-types.ts b/packages/taler-util/src/wallet-types.ts index accab746f..d49182e26 100644 --- a/packages/taler-util/src/wallet-types.ts +++ b/packages/taler-util/src/wallet-types.ts @@ -2673,3 +2673,15 @@ export interface RecoverStoredBackupRequest { export interface DeleteStoredBackupRequest { name: string; } + +export const codecForDeleteStoredBackupRequest = + (): Codec => + buildCodecForObject() + .property("name", codecForString()) + .build("DeleteStoredBackupRequest"); + +export const codecForRecoverStoredBackupRequest = + (): Codec => + buildCodecForObject() + .property("name", codecForString()) + .build("RecoverStoredBackupRequest"); diff --git a/packages/taler-wallet-cli/src/index.ts b/packages/taler-wallet-cli/src/index.ts index 36e7f7768..a0f44fb41 100644 --- a/packages/taler-wallet-cli/src/index.ts +++ b/packages/taler-wallet-cli/src/index.ts @@ -883,7 +883,7 @@ backupCli.subcommand("exportDb", "export-db").action(async (args) => { }); }); -backupCli.subcommand("storeBackup", "store-backup").action(async (args) => { +backupCli.subcommand("storeBackup", "store").action(async (args) => { await withWallet(args, async (wallet) => { const resp = await wallet.client.call( WalletApiOperation.CreateStoredBackup, @@ -893,6 +893,46 @@ backupCli.subcommand("storeBackup", "store-backup").action(async (args) => { }); }); +backupCli.subcommand("storeBackup", "list-stored").action(async (args) => { + await withWallet(args, async (wallet) => { + const resp = await wallet.client.call( + WalletApiOperation.ListStoredBackups, + {}, + ); + console.log(JSON.stringify(resp, undefined, 2)); + }); +}); + +backupCli + .subcommand("storeBackup", "delete-stored") + .requiredArgument("name", clk.STRING) + .action(async (args) => { + await withWallet(args, async (wallet) => { + const resp = await wallet.client.call( + WalletApiOperation.DeleteStoredBackup, + { + name: args.storeBackup.name, + }, + ); + console.log(JSON.stringify(resp, undefined, 2)); + }); + }); + +backupCli + .subcommand("recoverBackup", "recover-stored") + .requiredArgument("name", clk.STRING) + .action(async (args) => { + await withWallet(args, async (wallet) => { + const resp = await wallet.client.call( + WalletApiOperation.RecoverStoredBackup, + { + name: args.recoverBackup.name, + }, + ); + console.log(JSON.stringify(resp, undefined, 2)); + }); + }); + backupCli.subcommand("importDb", "import-db").action(async (args) => { await withWallet(args, async (wallet) => { const dumpRaw = await read(process.stdin); diff --git a/packages/taler-wallet-core/src/db.ts b/packages/taler-wallet-core/src/db.ts index a642c0203..b9d86eb25 100644 --- a/packages/taler-wallet-core/src/db.ts +++ b/packages/taler-wallet-core/src/db.ts @@ -22,6 +22,7 @@ import { IDBDatabase, IDBFactory, IDBObjectStore, + IDBRequest, IDBTransaction, structuredEncapsulate, } from "@gnu-taler/idb-bridge"; @@ -59,6 +60,7 @@ import { Logger, CoinPublicKeyString, TalerPreciseTimestamp, + j2s, } from "@gnu-taler/taler-util"; import { DbAccess, @@ -117,7 +119,8 @@ export const TALER_WALLET_META_DB_NAME = "taler-wallet-meta"; /** * Stored backups, mainly created when manually importing a backup. */ -export const TALER_WALLET_STORED_BACKUPS_DB_NAME = "taler-wallet-stored-backups"; +export const TALER_WALLET_STORED_BACKUPS_DB_NAME = + "taler-wallet-stored-backups"; export const CURRENT_DB_CONFIG_KEY = "currentMainDbName"; @@ -2833,11 +2836,10 @@ export async function exportSingleDb( dbName, undefined, () => { - // May not happen, since we're not requesting a specific version - throw Error("unexpected version change"); + logger.info(`unexpected onversionchange in exportSingleDb of ${dbName}`); }, () => { - logger.info("unexpected onupgradeneeded"); + logger.info(`unexpected onupgradeneeded in exportSingleDb of ${dbName}`); }, ); @@ -2849,7 +2851,7 @@ export async function exportSingleDb( return new Promise((resolve, reject) => { const tx = myDb.transaction(Array.from(myDb.objectStoreNames)); tx.addEventListener("complete", () => { - myDb.close(); + //myDb.close(); resolve(singleDbDump); }); // tslint:disable-next-line:prefer-for-of @@ -2885,6 +2887,7 @@ export async function exportSingleDb( if (store.keyPath == null) { rec.key = structuredEncapsulate(cursor.key); } + storeDump.records.push(rec); cursor.continue(); } }); @@ -2913,21 +2916,22 @@ async function recoverFromDump( db: IDBDatabase, dbDump: DbDumpDatabase, ): Promise { - return new Promise((resolve, reject) => { - const tx = db.transaction(Array.from(db.objectStoreNames), "readwrite"); - tx.addEventListener("complete", () => { - resolve(); - }); - for (let i = 0; i < db.objectStoreNames.length; i++) { - const name = db.objectStoreNames[i]; - const storeDump = dbDump.stores[name]; - if (!storeDump) continue; - for (let rec of storeDump.records) { - tx.objectStore(name).put(rec.value, rec.key); - } + const tx = db.transaction(Array.from(db.objectStoreNames), "readwrite"); + const txProm = promiseFromTransaction(tx); + const storeNames = db.objectStoreNames; + for (let i = 0; i < storeNames.length; i++) { + const name = db.objectStoreNames[i]; + const storeDump = dbDump.stores[name]; + if (!storeDump) continue; + await promiseFromRequest(tx.objectStore(name).clear()); + logger.info(`importing ${storeDump.records.length} records into ${name}`); + for (let rec of storeDump.records) { + await promiseFromRequest(tx.objectStore(name).put(rec.value, rec.key)); + logger.info("importing record done"); } - tx.commit(); - }); + } + tx.commit(); + return await txProm; } function checkDbDump(x: any): x is DbDump { @@ -3184,6 +3188,17 @@ function promiseFromTransaction(transaction: IDBTransaction): Promise { }); } +export function promiseFromRequest(request: IDBRequest): Promise { + return new Promise((resolve, reject) => { + request.onsuccess = () => { + resolve(request.result); + }; + request.onerror = () => { + reject(request.error); + }; + }); +} + /** * Purge all data in the given database. */ diff --git a/packages/taler-wallet-core/src/host-impl.node.ts b/packages/taler-wallet-core/src/host-impl.node.ts index 0b6539306..0626b9254 100644 --- a/packages/taler-wallet-core/src/host-impl.node.ts +++ b/packages/taler-wallet-core/src/host-impl.node.ts @@ -52,7 +52,6 @@ interface MakeDbResult { async function makeFileDb( args: DefaultNodeWalletArgs = {}, ): Promise { - BridgeIDBFactory.enableTracing = false; const myBackend = new MemoryBackend(); myBackend.enableTracing = false; const storagePath = args.persistentStoragePath; @@ -141,7 +140,10 @@ export async function createNativeWalletHost2( let dbResp: MakeDbResult; - if (args.persistentStoragePath &&args.persistentStoragePath.endsWith(".json")) { + if ( + args.persistentStoragePath && + args.persistentStoragePath.endsWith(".json") + ) { logger.info("using legacy file-based DB backend"); dbResp = await makeFileDb(args); } else { diff --git a/packages/taler-wallet-core/src/wallet.ts b/packages/taler-wallet-core/src/wallet.ts index 626409dd6..5666d67e0 100644 --- a/packages/taler-wallet-core/src/wallet.ts +++ b/packages/taler-wallet-core/src/wallet.ts @@ -121,6 +121,11 @@ import { GetCurrencyInfoResponse, codecForGetCurrencyInfoRequest, CreateStoredBackupResponse, + StoredBackupList, + codecForDeleteStoredBackupRequest, + DeleteStoredBackupRequest, + RecoverStoredBackupRequest, + codecForRecoverStoredBackupRequest, } from "@gnu-taler/taler-util"; import { HttpRequestLibrary, @@ -1041,6 +1046,57 @@ async function createStoredBackup( }; } +async function listStoredBackups( + ws: InternalWalletState, +): Promise { + const storedBackups: StoredBackupList = { + storedBackups: [], + }; + const backupsDb = await openStoredBackupsDatabase(ws.idb); + await backupsDb.mktxAll().runReadWrite(async (tx) => { + await tx.backupMeta.iter().forEach((x) => { + storedBackups.storedBackups.push({ + name: x.name, + }); + }); + }); + return storedBackups; +} + +async function deleteStoredBackup( + ws: InternalWalletState, + req: DeleteStoredBackupRequest, +): Promise { + const backupsDb = await openStoredBackupsDatabase(ws.idb); + await backupsDb.mktxAll().runReadWrite(async (tx) => { + await tx.backupData.delete(req.name); + await tx.backupMeta.delete(req.name); + }); +} + +async function recoverStoredBackup( + ws: InternalWalletState, + req: RecoverStoredBackupRequest, +): Promise { + logger.info(`Recovering stored backup ${req.name}`); + const { name } = req; + const backupsDb = await openStoredBackupsDatabase(ws.idb); + const bd = await backupsDb.mktxAll().runReadWrite(async (tx) => { + const backupMeta = tx.backupMeta.get(name); + if (!backupMeta) { + throw Error("backup not found"); + } + const backupData = await tx.backupData.get(name); + if (!backupData) { + throw Error("no backup data (DB corrupt)"); + } + return backupData; + }); + logger.info(`backup found, now importing`); + await importDb(ws.db.idbHandle(), bd); + logger.info(`import done`); +} + /** * Implementation of the "wallet-core" API. */ @@ -1059,12 +1115,18 @@ async function dispatchRequestInternal( switch (operation) { case WalletApiOperation.CreateStoredBackup: return createStoredBackup(ws); - case WalletApiOperation.DeleteStoredBackup: + case WalletApiOperation.DeleteStoredBackup: { + const req = codecForDeleteStoredBackupRequest().decode(payload); + await deleteStoredBackup(ws, req); return {}; + } case WalletApiOperation.ListStoredBackups: + return listStoredBackups(ws); + case WalletApiOperation.RecoverStoredBackup: { + const req = codecForRecoverStoredBackupRequest().decode(payload); + await recoverStoredBackup(ws, req); return {}; - case WalletApiOperation.RecoverStoredBackup: - return {}; + } case WalletApiOperation.InitWallet: { logger.trace("initializing wallet"); ws.initCalled = true;