implement infrastructure for future DB migrations via backup

packages/taler-wallet-cli/src/integrationtests/harness.ts

@@ -78,6 +78,7 @@ import {
   AcceptTipRequest,
   AbortPayWithRefundRequest,
   handleWorkerError,
+  openPromise,
 } from "taler-wallet-core";
 import { URL } from "url";
 import axios, { AxiosError } from "axios";
@@ -94,7 +95,6 @@ import {
 import { ApplyRefundResponse } from "taler-wallet-core";
 import { PendingOperationsResponse } from "taler-wallet-core";
 import { CoinConfig } from "./denomStructures";
-import { after } from "taler-wallet-core/src/util/timer";
 
 const exec = util.promisify(require("child_process").exec);
 
@@ -1114,11 +1114,14 @@ export class ExchangeService implements ExchangeServiceInterface {
       `exchange-httpd-${this.name}`,
     );
 
+    await this.pingUntilAvailable();
     await this.keyup();
   }
 
   async pingUntilAvailable(): Promise<void> {
-    const url = `http://localhost:${this.exchangeConfig.httpPort}/keys`;
+    // We request /management/keys, since /keys can block
+    // when we didn't do the key setup yet.
+    const url = `http://localhost:${this.exchangeConfig.httpPort}/management/keys`;
     await pingProc(this.exchangeHttpProc, url, `exchange (${this.name})`);
   }
 }
@@ -1449,10 +1452,14 @@ export async function runTestWithState(
 ): Promise<TestRunResult> {
   const startMs = new Date().getTime();
 
-  const handleSignal = () => {
+  const p = openPromise();
+  let status: TestStatus;
+
+  const handleSignal = (s: string) => {
     gc.shutdownSync();
-    console.warn("**** received fatal signal, shutting down test harness");
-    process.exit(1);
+    console.warn("**** received fatal proces event, shutting down test harness");
+    status = "fail";
+    p.reject(Error("caught signal"));
   };
 
   process.on("SIGINT", handleSignal);
@@ -1460,10 +1467,9 @@ export async function runTestWithState(
   process.on("unhandledRejection", handleSignal);
   process.on("uncaughtException", handleSignal);
 
-  let status: TestStatus;
   try {
     console.log("running test in directory", gc.testDir);
-    await testMain(gc);
+    await Promise.race([testMain(gc), p.promise]);
     status = "pass";
   } catch (e) {
     console.error("FATAL: test failed with exception", e);

packages/taler-wallet-cli/src/integrationtests/testrunner.ts

@@ -66,12 +66,13 @@ const allTests: TestMainFunction[] = [
   runMerchantLongpollingTest,
   runMerchantRefundApiTest,
   runPayAbortTest,
-  runPayPaidTest,
   runPaymentClaimTest,
   runPaymentFaultTest,
   runPaymentIdempotencyTest,
   runPaymentMultipleTest,
+  runPaymentTest,
   runPaymentTransientTest,
+  runPayPaidTest,
   runPaywallFlowTest,
   runRefundAutoTest,
   runRefundGoneTest,
@@ -82,10 +83,9 @@ const allTests: TestMainFunction[] = [
   runTimetravelWithdrawTest,
   runTippingTest,
   runWallettestingTest,
+  runTestWithdrawalManualTest,
   runWithdrawalAbortBankTest,
   runWithdrawalBankIntegratedTest,
-  runWallettestingTest,
-  runPaymentTest,
 ];
 
 export interface TestRunSpec {
@@ -166,7 +166,12 @@ export async function runTests(spec: TestRunSpec) {
     JSON.stringify({ testResults }, undefined, 2),
   );
   console.log(`See ${resultsFile} for details`);
+  console.log(`Skipped: ${numSkip}/${numTotal}`);
+  console.log(`Failed: ${numFail}/${numTotal}`);
   console.log(`Passed: ${numPass}/${numTotal}`);
+  if (numPass < numTotal - numSkip) {
+    process.exit(1);
+  }
 }
 
 export function getTestInfo(): TestInfo[] {

packages/taler-wallet-core/src/db.ts

@@ -1,5 +1,11 @@
-import { Stores } from "./types/dbTypes";
-import { openDatabase, Database, Store, Index } from "./util/query";
+import { MetaStores, Stores } from "./types/dbTypes";
+import {
+  openDatabase,
+  Database,
+  Store,
+  Index,
+  AnyStoreMap,
+} from "./util/query";
 import {
   IDBFactory,
   IDBDatabase,
@@ -14,7 +20,11 @@ import { Logger } from "./util/logging";
  * for all previous versions must be written, which should be
  * avoided.
  */
-const TALER_DB_NAME = "taler-wallet-prod-v1";
+const TALER_DB_NAME = "taler-wallet-main-v2";
+
+const TALER_META_DB_NAME = "taler-wallet-meta";
+
+const CURRENT_DB_CONFIG_KEY = "currentMainDbName";
 
 /**
  * Current database minor version, should be incremented
@@ -23,34 +33,25 @@ const TALER_DB_NAME = "taler-wallet-prod-v1";
  * backwards-compatible way or object stores and indices
  * are added.
  */
-export const WALLET_DB_MINOR_VERSION = 3;
+export const WALLET_DB_MINOR_VERSION = 1;
 
 const logger = new Logger("db.ts");
 
-/**
- * Return a promise that resolves
- * to the taler wallet db.
- */
-export function openTalerDatabase(
-  idbFactory: IDBFactory,
-  onVersionChange: () => void,
-): Promise<IDBDatabase> {
-  const onUpgradeNeeded = (
+function upgradeFromStoreMap(
+  storeMap: AnyStoreMap,
   db: IDBDatabase,
   oldVersion: number,
   newVersion: number,
   upgradeTransaction: IDBTransaction,
-  ): void => {
+): void {
   if (oldVersion === 0) {
-      for (const n in Stores) {
-        if ((Stores as any)[n] instanceof Store) {
-          const si: Store<string, any> = (Stores as any)[n];
+    for (const n in storeMap) {
+      if ((storeMap as any)[n] instanceof Store) {
+        const si: Store<string, any> = (storeMap as any)[n];
         const s = db.createObjectStore(si.name, si.storeParams);
         for (const indexName in si as any) {
           if ((si as any)[indexName] instanceof Index) {
-              const ii: Index<string, string, any, any> = (si as any)[
-                indexName
-              ];
+            const ii: Index<string, string, any, any> = (si as any)[indexName];
             s.createIndex(ii.indexName, ii.keyPath, ii.options);
           }
         }
@@ -86,15 +87,80 @@ export function openTalerDatabase(
       }
     }
   }
-  };
+}
 
-  return openDatabase(
+function onTalerDbUpgradeNeeded(
+  db: IDBDatabase,
+  oldVersion: number,
+  newVersion: number,
+  upgradeTransaction: IDBTransaction,
+) {
+  upgradeFromStoreMap(Stores, db, oldVersion, newVersion, upgradeTransaction);
+}
+
+function onMetaDbUpgradeNeeded(
+  db: IDBDatabase,
+  oldVersion: number,
+  newVersion: number,
+  upgradeTransaction: IDBTransaction,
+) {
+  upgradeFromStoreMap(
+    MetaStores,
+    db,
+    oldVersion,
+    newVersion,
+    upgradeTransaction,
+  );
+}
+
+/**
+ * Return a promise that resolves
+ * to the taler wallet db.
+ */
+export async function openTalerDatabase(
+  idbFactory: IDBFactory,
+  onVersionChange: () => void,
+): Promise<Database<typeof Stores>> {
+  const metaDbHandle = await openDatabase(
+    idbFactory,
+    TALER_META_DB_NAME,
+    1,
+    () => {},
+    onMetaDbUpgradeNeeded,
+  );
+
+  const metaDb = new Database(metaDbHandle, MetaStores);
+  let currentMainVersion: string | undefined;
+  await metaDb.runWithWriteTransaction([MetaStores.metaConfig], async (tx) => {
+    const dbVersionRecord = await tx.get(
+      MetaStores.metaConfig,
+      CURRENT_DB_CONFIG_KEY,
+    );
+    if (!dbVersionRecord) {
+      currentMainVersion = TALER_DB_NAME;
+      await tx.put(MetaStores.metaConfig, {
+        key: CURRENT_DB_CONFIG_KEY,
+        value: TALER_DB_NAME,
+      });
+    } else {
+      currentMainVersion = dbVersionRecord.key;
+    }
+  });
+
+  if (currentMainVersion !== TALER_DB_NAME) {
+    // In the future, the migration logic will be implemented here.
+    throw Error(`migration from database ${currentMainVersion} not supported`);
+  }
+
+  const mainDbHandle = await openDatabase(
     idbFactory,
     TALER_DB_NAME,
     WALLET_DB_MINOR_VERSION,
     onVersionChange,
-    onUpgradeNeeded,
+    onTalerDbUpgradeNeeded,
   );
+
+  return new Database(mainDbHandle, Stores);
 }
 
 export function deleteTalerDatabase(idbFactory: IDBFactory): void {

packages/taler-wallet-core/src/headless/helpers.ts

@@ -34,6 +34,7 @@ import { NodeHttpLib } from "./NodeHttpLib";
 import { Logger } from "../util/logging";
 import { SynchronousCryptoWorkerFactory } from "../crypto/workers/synchronousWorker";
 import type { IDBFactory } from "idb-bridge/lib/idbtypes";
+import { Stores } from "../types/dbTypes";
 
 const logger = new Logger("headless/helpers.ts");
 
@@ -149,9 +150,7 @@ export async function getDefaultNodeWallet(
     workerFactory = new SynchronousCryptoWorkerFactory();
   }
 
-  const dbWrap = new Database(myDb);
-
-  const w = new Wallet(dbWrap, myHttpLib, workerFactory);
+  const w = new Wallet(myDb, myHttpLib, workerFactory);
   if (args.notifyHandler) {
     w.addNotificationListener(args.notifyHandler);
   }

packages/taler-wallet-core/src/index.ts

@@ -34,6 +34,7 @@ export {
 export * from "./operations/versions";
 
 export * from "./db";
+export * from "./types/dbTypes";
 
 // Internationalization
 export * from "./i18n";

packages/taler-wallet-core/src/operations/exchanges.ts

@@ -29,6 +29,7 @@ import {
   DenominationStatus,
   WireFee,
   ExchangeUpdateReason,
+  MetaStores,
 } from "../types/dbTypes";
 import { canonicalizeBaseUrl } from "../util/helpers";
 import * as Amounts from "../util/amounts";

packages/taler-wallet-core/src/operations/state.ts

@@ -23,6 +23,7 @@ import { PendingOperationsResponse } from "../types/pendingTypes";
 import { WalletNotification } from "../types/notifications";
 import { Database } from "../util/query";
 import { openPromise, OpenedPromise } from "../util/promiseUtils";
+import { Stores } from "../types/dbTypes";
 
 type NotificationListener = (n: WalletNotification) => void;
 
@@ -59,7 +60,7 @@ export class InternalWalletState {
     // the actual value nullable. 
     // Check if we are in a DB migration / garbage collection
     // and throw an error in that case.
-    public db: Database,
+    public db: Database<typeof Stores>,
     public http: HttpRequestLibrary,
     cryptoWorkerFactory: CryptoWorkerFactory,
   ) {

packages/taler-wallet-core/src/types/dbTypes.ts

@@ -1591,9 +1591,6 @@ class TipsStore extends Store<"tips", TipRecord> {
     this,
     "tipsByMerchantTipIdAndOriginIndex",
     ["merchantTipId", "merchantBaseUrl"],
-    {
-      versionAdded: 2,
-    },
   );
 }
 
@@ -1657,7 +1654,7 @@ class BackupProvidersStore extends Store<
   BackupProviderRecord
 > {
   constructor() {
-    super("backupProviders", { keyPath: "baseUrl", versionAdded: 3 });
+    super("backupProviders", { keyPath: "baseUrl" });
   }
 }
 
@@ -1688,3 +1685,13 @@ export const Stores = {
   bankWithdrawUris: new BankWithdrawUrisStore(),
   backupProviders: new BackupProvidersStore(),
 };
+
+export class MetaConfigStore extends Store<"metaConfig", ConfigRecord<any>> {
+  constructor() {
+    super("metaConfig", { keyPath: "key" });
+  }
+}
+
+export const MetaStores = {
+  metaConfig: new MetaConfigStore(),
+};

packages/taler-wallet-core/src/util/query.ts

@@ -269,6 +269,8 @@ class ResultStream<T> {
   }
 }
 
+export type AnyStoreMap = { [s: string]: Store<any, any> };
+
 type StoreName<S> = S extends Store<infer N, any> ? N : never;
 type StoreContent<S> = S extends Store<any, infer R> ? R : never;
 type IndexRecord<Ind> = Ind extends Index<any, any, any, infer R> ? R : never;
@@ -462,8 +464,7 @@ export class Index<
 }
 
 /**
- * Return a promise that resolves
- * to the taler wallet db.
+ * Return a promise that resolves to the opened IndexedDB database.
  */
 export function openDatabase(
   idbFactory: IDBFactory,
@@ -480,7 +481,7 @@ export function openDatabase(
   return new Promise<IDBDatabase>((resolve, reject) => {
     const req = idbFactory.open(databaseName, databaseVersion);
     req.onerror = (e) => {
-      logger.error("taler database error", e);
+      logger.error("database error", e);
       reject(new Error("database error"));
     };
     req.onsuccess = (e) => {
@@ -508,8 +509,8 @@ export function openDatabase(
   });
 }
 
-export class Database {
-  constructor(private db: IDBDatabase) {}
+export class Database<StoreMap extends AnyStoreMap> {
+  constructor(private db: IDBDatabase, stores: StoreMap) {}
 
   static deleteDatabase(idbFactory: IDBFactory, dbName: string): void {
     idbFactory.deleteDatabase(dbName);
@@ -571,10 +572,10 @@ export class Database {
     });
   }
 
-  async get<N extends string, T>(
-    store: Store<N, T>,
+  async get<N extends keyof StoreMap, S extends StoreMap[N]>(
+    store: S,
     key: IDBValidKey,
-  ): Promise<T | undefined> {
+  ): Promise<StoreContent<S> | undefined> {
     const tx = this.db.transaction([store.name], "readonly");
     const req = tx.objectStore(store.name).get(key);
     const v = await requestToPromise(req);
@@ -634,14 +635,22 @@ export class Database {
     return new ResultStream<IndexRecord<Ind>>(req);
   }
 
-  async runWithReadTransaction<T, StoreTypes extends Store<string, any>>(
+  async runWithReadTransaction<
+    T,
+    N extends keyof StoreMap,
+    StoreTypes extends StoreMap[N]
+  >(
     stores: StoreTypes[],
     f: (t: TransactionHandle<StoreTypes>) => Promise<T>,
   ): Promise<T> {
     return runWithTransaction<T, StoreTypes>(this.db, stores, f, "readonly");
   }
 
-  async runWithWriteTransaction<T, StoreTypes extends Store<string, any>>(
+  async runWithWriteTransaction<
+    T,
+    N extends keyof StoreMap,
+    StoreTypes extends StoreMap[N]
+  >(
     stores: StoreTypes[],
     f: (t: TransactionHandle<StoreTypes>) => Promise<T>,
   ): Promise<T> {

packages/taler-wallet-core/src/wallet.ts

@@ -24,7 +24,7 @@
  */
 import { CryptoWorkerFactory } from "./crypto/workers/cryptoApi";
 import { HttpRequestLibrary } from "./util/http";
-import { Database } from "./util/query";
+import { Database, Store } from "./util/query";
 
 import { Amounts, AmountJson } from "./util/amounts";
 
@@ -52,6 +52,7 @@ import {
   ReserveRecordStatus,
   CoinSourceType,
   RefundState,
+  MetaStores,
 } from "./types/dbTypes";
 import { CoinDumpJson, WithdrawUriInfoResponse } from "./types/talerTypes";
 import {
@@ -200,12 +201,12 @@ export class Wallet {
   private stopped = false;
   private memoRunRetryLoop = new AsyncOpMemoSingle<void>();
 
-  get db(): Database {
+  get db(): Database<typeof Stores> {
     return this.ws.db;
   }
 
   constructor(
-    db: Database,
+    db: Database<typeof Stores>,
     http: HttpRequestLibrary,
     cryptoWorkerFactory: CryptoWorkerFactory,
   ) {

packages/taler-wallet-webextension/src/wxBackend.ts

@@ -24,7 +24,6 @@
  * Imports.
  */
 import { isFirefox, getPermissionsApi } from "./compat";
-import MessageSender = chrome.runtime.MessageSender;
 import { extendedPermissions } from "./permissions";
 
 import {
@@ -40,6 +39,7 @@ import {
   CoreApiResponse,
   WalletDiagnostics,
   CoreApiResponseSuccess,
+  Stores,
 } from "taler-wallet-core";
 import { BrowserHttpLib } from "./browserHttpLib";
 import { BrowserCryptoWorkerFactory } from "./browserCryptoWorkerFactory";
@@ -50,7 +50,7 @@ import { BrowserCryptoWorkerFactory } from "./browserCryptoWorkerFactory";
  */
 let currentWallet: Wallet | undefined;
 
-let currentDatabase: IDBDatabase | undefined;
+let currentDatabase: Database<typeof Stores> | undefined;
 
 /**
  * Last version if an outdated DB, if applicable.
@@ -135,7 +135,7 @@ async function dispatch(
         setupHeaderListener();
         r = wrapResponse({ newValue: true });
       } else {
-        await new Promise((resolve, reject) => {
+        await new Promise<void>((resolve, reject) => {
           getPermissionsApi().remove(extendedPermissions, (rem) => {
             console.log("permissions removed:", rem);
             resolve();
@@ -246,7 +246,7 @@ async function reinitWallet(): Promise<void> {
   const http = new BrowserHttpLib();
   console.log("setting wallet");
   const wallet = new Wallet(
-    new Database(currentDatabase),
+    currentDatabase,
     http,
     new BrowserCryptoWorkerFactory(),
   );

pnpm-lock.yaml

@@ -31,28 +31,6 @@ importers:
     specifiers:
       '@types/node': ^14.14.7
       typescript: ^4.0.5
-  packages/taler-integrationtests:
-    dependencies:
-      axios: 0.21.0
-      taler-wallet-core: link:../taler-wallet-core
-      tslib: 2.0.3
-    devDependencies:
-      esm: 3.2.25
-      nyc: 15.1.0
-      prettier: 2.1.2
-      source-map-support: 0.5.19
-      ts-node: 9.0.0_typescript@4.0.5
-      typescript: 4.0.5
-    specifiers:
-      axios: ^0.21.0
-      esm: ^3.2.25
-      nyc: ^15.1.0
-      prettier: ^2.1.2
-      source-map-support: ^0.5.19
-      taler-wallet-core: workspace:*
-      ts-node: ^9.0.0
-      tslib: ^2.0.3
-      typescript: ^4.0.5
   packages/taler-wallet-android:
     dependencies:
       taler-wallet-core: link:../taler-wallet-core
@@ -1031,10 +1009,6 @@ packages:
     dev: true
     resolution:
       integrity: sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=
-  /arg/4.1.3:
-    dev: true
-    resolution:
-      integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
   /argparse/1.0.10:
     dependencies:
       sprintf-js: 1.0.3
@@ -1737,12 +1711,6 @@ packages:
       node: '>=10'
     resolution:
       integrity: sha512-1shh9DQ23L16oXSZKB2JxpL7iMy2E0S9d517ptA1P8iw0alkPtQcrKH7ru31rYtKwF499HkTu+DRzq3TCKDFRQ==
-  /diff/4.0.2:
-    dev: true
-    engines:
-      node: '>=0.3.1'
-    resolution:
-      integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
   /dir-glob/3.0.1:
     dependencies:
       path-type: 4.0.0
@@ -3352,10 +3320,6 @@ packages:
       node: '>=8'
     resolution:
       integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==
-  /make-error/1.3.6:
-    dev: true
-    resolution:
-      integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==
   /map-age-cleaner/0.1.3:
     dependencies:
       p-defer: 1.0.0
@@ -4788,22 +4752,6 @@ packages:
       node: '>=0.10.0'
     resolution:
       integrity: sha1-n5up2e+odkw4dpi8v+sshI8RrbM=
-  /ts-node/9.0.0_typescript@4.0.5:
-    dependencies:
-      arg: 4.1.3
-      diff: 4.0.2
-      make-error: 1.3.6
-      source-map-support: 0.5.19
-      typescript: 4.0.5
-      yn: 3.1.1
-    dev: true
-    engines:
-      node: '>=10.0.0'
-    hasBin: true
-    peerDependencies:
-      typescript: '>=2.7'
-    resolution:
-      integrity: sha512-/TqB4SnererCDR/vb4S/QvSZvzQMJN8daAslg7MeaiHvD8rDZsSfXmNeNumyZZzMned72Xoq/isQljYSt8Ynfg==
   /tsconfig-paths/3.9.0:
     dependencies:
       '@types/json5': 0.0.29
@@ -5144,12 +5092,6 @@ packages:
       node: '>=10'
     resolution:
       integrity: sha512-hAD1RcFP/wfgfxgMVswPE+z3tlPFtxG8/yWUrG2i17sTWGCGqWnxKcLTF4cUKDUK8fzokwsmO9H0TDkRbMHy8w==
-  /yn/3.1.1:
-    dev: true
-    engines:
-      node: '>=6'
-    resolution:
-      integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==
   /z-schema/3.18.4:
     dependencies:
       lodash.get: 4.4.2