wallet-core: implement and test stored backups

packages/idb-bridge/src/SqliteBackend.ts

@@ -1882,7 +1882,7 @@ export class SqliteBackend implements Backend {
     }
   }
 
-  clearObjectStore(
+  async clearObjectStore(
     btx: DatabaseTransaction,
     objectStoreName: string,
   ): Promise<void> {
@@ -1906,7 +1906,21 @@ export class SqliteBackend implements Backend {
       );
     }
 
-    throw new Error("Method not implemented.");
+    this._prep(sqlClearObjectStore).run({
+      object_store_id: scopeInfo.objectStoreId,
+    });
+
+    for (const index of scopeInfo.indexMap.values()) {
+      let stmt: Sqlite3Statement;
+      if (index.unique) {
+        stmt = this._prep(sqlClearUniqueIndexData);
+      } else {
+        stmt = this._prep(sqlClearIndexData);
+      }
+      stmt.run({
+        index_id: index.indexId,
+      });
+    }
   }
 }
 
@@ -1963,6 +1977,15 @@ CREATE TABLE IF NOT EXISTS unique_index_data
 );
 `;
 
+const sqlClearObjectStore = `
+DELETE FROM object_data WHERE object_store_id=$object_store_id`;
+
+const sqlClearIndexData = `
+DELETE FROM index_data WHERE index_id=$index_id`;
+
+const sqlClearUniqueIndexData = `
+DELETE FROM unique_index_data WHERE index_id=$index_id`;
+
 const sqlListDatabases = `
 SELECT name, version FROM databases;
 `;

packages/idb-bridge/src/bridge-idb.ts

@@ -735,7 +735,9 @@ export class BridgeIDBDatabase extends FakeEventTarget implements IDBDatabase {
     }
 
     if (this._closePending) {
-      throw new InvalidStateError();
+      throw new InvalidStateError(
+        `tried to start transaction on ${this._name}, but a close is pending`,
+      );
     }
 
     if (!Array.isArray(storeNames)) {
@@ -930,6 +932,9 @@ export class BridgeIDBFactory {
         // http://www.w3.org/TR/2015/REC-IndexedDB-20150108/#dfn-steps-for-running-a-versionchange-transaction
 
         for (const otherConn of this.connections) {
+          if (otherConn._name != db._name) {
+            continue;
+          }
           if (otherConn._closePending) {
             continue;
           }

packages/taler-harness/src/integrationtests/test-stored-backups.ts

@@ -0,0 +1,110 @@
+/*
+ This file is part of GNU Taler
+ (C) 2023 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+ */
+
+/**
+ * Imports.
+ */
+import { WalletApiOperation } from "@gnu-taler/taler-wallet-core";
+import { GlobalTestState } from "../harness/harness.js";
+import {
+  withdrawViaBankV2,
+  makeTestPaymentV2,
+  useSharedTestkudosEnvironment,
+} from "../harness/helpers.js";
+
+/**
+ * Test stored backup wallet-core API.
+ */
+export async function runStoredBackupsTest(t: GlobalTestState) {
+  // Set up test environment
+
+  const { walletClient, bank, exchange, merchant } =
+    await useSharedTestkudosEnvironment(t);
+
+  // Withdraw digital cash into the wallet.
+
+  await withdrawViaBankV2(t, {
+    walletClient,
+    bank,
+    exchange,
+    amount: "TESTKUDOS:20",
+  });
+
+  await walletClient.call(WalletApiOperation.TestingWaitTransactionsFinal, {});
+
+  const sb1Resp = await walletClient.call(
+    WalletApiOperation.CreateStoredBackup,
+    {},
+  );
+  const sbList = await walletClient.call(
+    WalletApiOperation.ListStoredBackups,
+    {},
+  );
+  t.assertTrue(sbList.storedBackups.length === 1);
+  t.assertTrue(sbList.storedBackups[0].name === sb1Resp.name);
+
+  const order = {
+    summary: "Buy me!",
+    amount: "TESTKUDOS:5",
+    fulfillment_url: "taler://fulfillment-success/thx",
+  };
+
+  await makeTestPaymentV2(t, { walletClient, merchant, order });
+  await walletClient.call(WalletApiOperation.TestingWaitTransactionsFinal, {});
+
+  const txn1 = await walletClient.call(WalletApiOperation.GetTransactions, {});
+  t.assertDeepEqual(txn1.transactions.length, 2);
+
+  // Recover from the stored backup now.
+
+  const sb2Resp = await walletClient.call(
+    WalletApiOperation.CreateStoredBackup,
+    {},
+  );
+
+  console.log("recovering backup");
+
+  await walletClient.call(WalletApiOperation.RecoverStoredBackup, {
+    name: sb1Resp.name,
+  });
+
+  console.log("first recovery done");
+
+  // Recovery went well, now we can delete the backup
+  // of the old database we stored before importing.
+  {
+    const sbl1 = await walletClient.call(
+      WalletApiOperation.ListStoredBackups,
+      {},
+    );
+    t.assertTrue(sbl1.storedBackups.length === 2);
+
+    await walletClient.call(WalletApiOperation.DeleteStoredBackup, {
+      name: sb1Resp.name,
+    });
+    const sbl2 = await walletClient.call(
+      WalletApiOperation.ListStoredBackups,
+      {},
+    );
+    t.assertTrue(sbl2.storedBackups.length === 1);
+  }
+
+  const txn2 = await walletClient.call(WalletApiOperation.GetTransactions, {});
+  // We only have the withdrawal after restoring
+  t.assertDeepEqual(txn2.transactions.length, 1);
+}
+
+runStoredBackupsTest.suites = ["wallet"];

packages/taler-harness/src/integrationtests/testrunner.ts

@@ -114,6 +114,7 @@ import { runSimplePaymentTest } from "./test-simple-payment.js";
 import { runTermOfServiceFormatTest } from "./test-tos-format.js";
 import { runExchangePurseTest } from "./test-exchange-purse.js";
 import { getSharedTestDir } from "../harness/helpers.js";
+import { runStoredBackupsTest } from "./test-stored-backups.js";
 
 /**
  * Test runner.
@@ -212,6 +213,7 @@ const allTests: TestMainFunction[] = [
   runWithdrawalFeesTest,
   runWithdrawalHugeTest,
   runTermOfServiceFormatTest,
+  runStoredBackupsTest,
 ];
 
 export interface TestRunSpec {

packages/taler-util/src/backup-types.ts

@@ -14,6 +14,8 @@
  GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
  */
 
+import { AmountString } from "./taler-types.js";
+
 export interface BackupRecovery {
   walletRootPriv: string;
   providers: {
@@ -21,3 +23,20 @@ export interface BackupRecovery {
     url: string;
   }[];
 }
+
+export class BackupBackupProviderTerms {
+  /**
+   * Last known supported protocol version.
+   */
+  supported_protocol_version: string;
+
+  /**
+   * Last known annual fee.
+   */
+  annual_fee: AmountString;
+
+  /**
+   * Last known storage limit.
+   */
+  storage_limit_in_megabytes: number;
+}

packages/taler-util/src/wallet-types.ts

@@ -2673,3 +2673,15 @@ export interface RecoverStoredBackupRequest {
 export interface DeleteStoredBackupRequest {
   name: string;
 }
+
+export const codecForDeleteStoredBackupRequest =
+  (): Codec<DeleteStoredBackupRequest> =>
+    buildCodecForObject<DeleteStoredBackupRequest>()
+      .property("name", codecForString())
+      .build("DeleteStoredBackupRequest");
+
+export const codecForRecoverStoredBackupRequest =
+  (): Codec<RecoverStoredBackupRequest> =>
+    buildCodecForObject<RecoverStoredBackupRequest>()
+      .property("name", codecForString())
+      .build("RecoverStoredBackupRequest");

packages/taler-wallet-cli/src/index.ts

@@ -883,7 +883,7 @@ backupCli.subcommand("exportDb", "export-db").action(async (args) => {
   });
 });
 
-backupCli.subcommand("storeBackup", "store-backup").action(async (args) => {
+backupCli.subcommand("storeBackup", "store").action(async (args) => {
   await withWallet(args, async (wallet) => {
     const resp = await wallet.client.call(
       WalletApiOperation.CreateStoredBackup,
@@ -893,6 +893,46 @@ backupCli.subcommand("storeBackup", "store-backup").action(async (args) => {
   });
 });
 
+backupCli.subcommand("storeBackup", "list-stored").action(async (args) => {
+  await withWallet(args, async (wallet) => {
+    const resp = await wallet.client.call(
+      WalletApiOperation.ListStoredBackups,
+      {},
+    );
+    console.log(JSON.stringify(resp, undefined, 2));
+  });
+});
+
+backupCli
+  .subcommand("storeBackup", "delete-stored")
+  .requiredArgument("name", clk.STRING)
+  .action(async (args) => {
+    await withWallet(args, async (wallet) => {
+      const resp = await wallet.client.call(
+        WalletApiOperation.DeleteStoredBackup,
+        {
+          name: args.storeBackup.name,
+        },
+      );
+      console.log(JSON.stringify(resp, undefined, 2));
+    });
+  });
+
+backupCli
+  .subcommand("recoverBackup", "recover-stored")
+  .requiredArgument("name", clk.STRING)
+  .action(async (args) => {
+    await withWallet(args, async (wallet) => {
+      const resp = await wallet.client.call(
+        WalletApiOperation.RecoverStoredBackup,
+        {
+          name: args.recoverBackup.name,
+        },
+      );
+      console.log(JSON.stringify(resp, undefined, 2));
+    });
+  });
+
 backupCli.subcommand("importDb", "import-db").action(async (args) => {
   await withWallet(args, async (wallet) => {
     const dumpRaw = await read(process.stdin);

packages/taler-wallet-core/src/db.ts

@@ -22,6 +22,7 @@ import {
   IDBDatabase,
   IDBFactory,
   IDBObjectStore,
+  IDBRequest,
   IDBTransaction,
   structuredEncapsulate,
 } from "@gnu-taler/idb-bridge";
@@ -59,6 +60,7 @@ import {
   Logger,
   CoinPublicKeyString,
   TalerPreciseTimestamp,
+  j2s,
 } from "@gnu-taler/taler-util";
 import {
   DbAccess,
@@ -117,7 +119,8 @@ export const TALER_WALLET_META_DB_NAME = "taler-wallet-meta";
 /**
  * Stored backups, mainly created when manually importing a backup.
  */
-export const TALER_WALLET_STORED_BACKUPS_DB_NAME = "taler-wallet-stored-backups";
+export const TALER_WALLET_STORED_BACKUPS_DB_NAME =
+  "taler-wallet-stored-backups";
 
 export const CURRENT_DB_CONFIG_KEY = "currentMainDbName";
 
@@ -2833,11 +2836,10 @@ export async function exportSingleDb(
     dbName,
     undefined,
     () => {
-      // May not happen, since we're not requesting a specific version
+      logger.info(`unexpected onversionchange in exportSingleDb of ${dbName}`);
-      throw Error("unexpected version change");
     },
     () => {
-      logger.info("unexpected onupgradeneeded");
+      logger.info(`unexpected onupgradeneeded in exportSingleDb of ${dbName}`);
     },
   );
 
@@ -2849,7 +2851,7 @@ export async function exportSingleDb(
   return new Promise((resolve, reject) => {
     const tx = myDb.transaction(Array.from(myDb.objectStoreNames));
     tx.addEventListener("complete", () => {
-      myDb.close();
+      //myDb.close();
       resolve(singleDbDump);
     });
     // tslint:disable-next-line:prefer-for-of
@@ -2885,6 +2887,7 @@ export async function exportSingleDb(
           if (store.keyPath == null) {
             rec.key = structuredEncapsulate(cursor.key);
           }
+          storeDump.records.push(rec);
           cursor.continue();
         }
       });
@@ -2913,21 +2916,22 @@ async function recoverFromDump(
   db: IDBDatabase,
   dbDump: DbDumpDatabase,
 ): Promise<void> {
-  return new Promise((resolve, reject) => {
   const tx = db.transaction(Array.from(db.objectStoreNames), "readwrite");
-    tx.addEventListener("complete", () => {
+  const txProm = promiseFromTransaction(tx);
-      resolve();
+  const storeNames = db.objectStoreNames;
-    });
+  for (let i = 0; i < storeNames.length; i++) {
-    for (let i = 0; i < db.objectStoreNames.length; i++) {
     const name = db.objectStoreNames[i];
     const storeDump = dbDump.stores[name];
     if (!storeDump) continue;
+    await promiseFromRequest(tx.objectStore(name).clear());
+    logger.info(`importing ${storeDump.records.length} records into ${name}`);
     for (let rec of storeDump.records) {
-        tx.objectStore(name).put(rec.value, rec.key);
+      await promiseFromRequest(tx.objectStore(name).put(rec.value, rec.key));
+      logger.info("importing record done");
     }
   }
   tx.commit();
-  });
+  return await txProm;
 }
 
 function checkDbDump(x: any): x is DbDump {
@@ -3184,6 +3188,17 @@ function promiseFromTransaction(transaction: IDBTransaction): Promise<void> {
   });
 }
 
+export function promiseFromRequest(request: IDBRequest): Promise<any> {
+  return new Promise((resolve, reject) => {
+    request.onsuccess = () => {
+      resolve(request.result);
+    };
+    request.onerror = () => {
+      reject(request.error);
+    };
+  });
+}
+
 /**
  * Purge all data in the given database.
  */

packages/taler-wallet-core/src/host-impl.node.ts

@@ -52,7 +52,6 @@ interface MakeDbResult {
 async function makeFileDb(
   args: DefaultNodeWalletArgs = {},
 ): Promise<MakeDbResult> {
-  BridgeIDBFactory.enableTracing = false;
   const myBackend = new MemoryBackend();
   myBackend.enableTracing = false;
   const storagePath = args.persistentStoragePath;
@@ -141,7 +140,10 @@ export async function createNativeWalletHost2(
 
   let dbResp: MakeDbResult;
 
-  if (args.persistentStoragePath &&args.persistentStoragePath.endsWith(".json")) {
+  if (
+    args.persistentStoragePath &&
+    args.persistentStoragePath.endsWith(".json")
+  ) {
     logger.info("using legacy file-based DB backend");
     dbResp = await makeFileDb(args);
   } else {

packages/taler-wallet-core/src/wallet.ts

@@ -121,6 +121,11 @@ import {
   GetCurrencyInfoResponse,
   codecForGetCurrencyInfoRequest,
   CreateStoredBackupResponse,
+  StoredBackupList,
+  codecForDeleteStoredBackupRequest,
+  DeleteStoredBackupRequest,
+  RecoverStoredBackupRequest,
+  codecForRecoverStoredBackupRequest,
 } from "@gnu-taler/taler-util";
 import {
   HttpRequestLibrary,
@@ -1041,6 +1046,57 @@ async function createStoredBackup(
   };
 }
 
+async function listStoredBackups(
+  ws: InternalWalletState,
+): Promise<StoredBackupList> {
+  const storedBackups: StoredBackupList = {
+    storedBackups: [],
+  };
+  const backupsDb = await openStoredBackupsDatabase(ws.idb);
+  await backupsDb.mktxAll().runReadWrite(async (tx) => {
+    await tx.backupMeta.iter().forEach((x) => {
+      storedBackups.storedBackups.push({
+        name: x.name,
+      });
+    });
+  });
+  return storedBackups;
+}
+
+async function deleteStoredBackup(
+  ws: InternalWalletState,
+  req: DeleteStoredBackupRequest,
+): Promise<void> {
+  const backupsDb = await openStoredBackupsDatabase(ws.idb);
+  await backupsDb.mktxAll().runReadWrite(async (tx) => {
+    await tx.backupData.delete(req.name);
+    await tx.backupMeta.delete(req.name);
+  });
+}
+
+async function recoverStoredBackup(
+  ws: InternalWalletState,
+  req: RecoverStoredBackupRequest,
+): Promise<void> {
+  logger.info(`Recovering stored backup ${req.name}`);
+  const { name } = req;
+  const backupsDb = await openStoredBackupsDatabase(ws.idb);
+  const bd = await backupsDb.mktxAll().runReadWrite(async (tx) => {
+    const backupMeta = tx.backupMeta.get(name);
+    if (!backupMeta) {
+      throw Error("backup not found");
+    }
+    const backupData = await tx.backupData.get(name);
+    if (!backupData) {
+      throw Error("no backup data (DB corrupt)");
+    }
+    return backupData;
+  });
+  logger.info(`backup found, now importing`);
+  await importDb(ws.db.idbHandle(), bd);
+  logger.info(`import done`);
+}
+
 /**
  * Implementation of the "wallet-core" API.
  */
@@ -1059,12 +1115,18 @@ async function dispatchRequestInternal<Op extends WalletApiOperation>(
   switch (operation) {
     case WalletApiOperation.CreateStoredBackup:
       return createStoredBackup(ws);
-    case WalletApiOperation.DeleteStoredBackup:
+    case WalletApiOperation.DeleteStoredBackup: {
+      const req = codecForDeleteStoredBackupRequest().decode(payload);
+      await deleteStoredBackup(ws, req);
       return {};
+    }
     case WalletApiOperation.ListStoredBackups:
+      return listStoredBackups(ws);
+    case WalletApiOperation.RecoverStoredBackup: {
+      const req = codecForRecoverStoredBackupRequest().decode(payload);
+      await recoverStoredBackup(ws, req);
       return {};
-    case WalletApiOperation.RecoverStoredBackup:
+    }
-      return {};
     case WalletApiOperation.InitWallet: {
       logger.trace("initializing wallet");
       ws.initCalled = true;