finish first complete end-to-end backup/sync test

packages/taler-wallet-cli/src/integrationtests/harness.ts

@@ -82,6 +82,7 @@ import {
   CreateDepositGroupResponse,
   TrackDepositGroupRequest,
   TrackDepositGroupResponse,
+  RecoveryLoadRequest,
 } from "@gnu-taler/taler-wallet-core";
 import { URL } from "url";
 import axios, { AxiosError } from "axios";
@@ -102,6 +103,7 @@ import { CoinConfig } from "./denomStructures";
 import {
   AddBackupProviderRequest,
   BackupInfo,
+  BackupRecovery,
 } from "@gnu-taler/taler-wallet-core/src/operations/backup";
 
 const exec = util.promisify(require("child_process").exec);
@@ -1887,6 +1889,22 @@ export class WalletCli {
     throw new OperationFailedError(resp.error);
   }
 
+  async exportBackupRecovery(): Promise<BackupRecovery> {
+    const resp = await this.apiRequest("exportBackupRecovery", {});
+    if (resp.type === "response") {
+      return resp.result as BackupRecovery;
+    }
+    throw new OperationFailedError(resp.error);
+  }
+
+  async importBackupRecovery(req: RecoveryLoadRequest): Promise<void> {
+    const resp = await this.apiRequest("importBackupRecovery", req);
+    if (resp.type === "response") {
+      return;
+    }
+    throw new OperationFailedError(resp.error);
+  }
+
   async runBackupCycle(): Promise<void> {
     const resp = await this.apiRequest("runBackupCycle", {});
     if (resp.type === "response") {

packages/taler-wallet-cli/src/integrationtests/sync.ts

@@ -19,7 +19,6 @@
  */
 import axios from "axios";
 import { Configuration, URL } from "@gnu-taler/taler-wallet-core";
-import { getRandomIban, getRandomString } from "./helpers";
 import * as fs from "fs";
 import * as util from "util";
 import {
@@ -87,6 +86,8 @@ export class SyncService {
     config.setString("sync", "port", `${sc.httpPort}`);
     config.setString("sync", "db", "postgres");
     config.setString("syncdb-postgres", "config", sc.database);
+    config.setString("sync", "payment_backend_url", sc.paymentBackendUrl);
+    config.setString("sync", "upload_limit_mb", `${sc.uploadLimitMb}`);
     config.write(cfgFilename);
 
     return new SyncService(gc, sc, cfgFilename);

packages/taler-wallet-cli/src/integrationtests/test-wallet-backup-basic.ts

@@ -17,9 +17,12 @@
 /**
  * Imports.
  */
-import { GlobalTestState, BankApi, BankAccessApi } from "./harness";
+import { GlobalTestState, BankApi, BankAccessApi, WalletCli } from "./harness";
-import { createSimpleTestkudosEnvironment } from "./helpers";
+import {
-import { codecForBalancesResponse } from "@gnu-taler/taler-wallet-core";
+  createSimpleTestkudosEnvironment,
+  makeTestPayment,
+  withdrawViaBank,
+} from "./helpers";
 import { SyncService } from "./sync";
 
 /**
@@ -28,7 +31,13 @@ import { SyncService } from "./sync";
 export async function runWalletBackupBasicTest(t: GlobalTestState) {
   // Set up test environment
 
-  const { commonDb, merchant, wallet, bank, exchange } = await createSimpleTestkudosEnvironment(t);
+  const {
+    commonDb,
+    merchant,
+    wallet,
+    bank,
+    exchange,
+  } = await createSimpleTestkudosEnvironment(t);
 
   const sync = await SyncService.create(t, {
     currency: "TESTKUDOS",
@@ -69,5 +78,48 @@ export async function runWalletBackupBasicTest(t: GlobalTestState) {
   {
     const bi = await wallet.getBackupInfo();
     console.log(bi);
+    t.assertDeepEqual(
+      bi.providers[0].paymentStatus.type,
+      "insufficient-balance",
+    );
+  }
+
+  await withdrawViaBank(t, { wallet, bank, exchange, amount: "TESTKUDOS:10" });
+
+  await wallet.runBackupCycle();
+
+  {
+    const bi = await wallet.getBackupInfo();
+    console.log(bi);
+  }
+
+  await withdrawViaBank(t, { wallet, bank, exchange, amount: "TESTKUDOS:5" });
+
+  await wallet.runBackupCycle();
+
+  {
+    const bi = await wallet.getBackupInfo();
+    console.log(bi);
+  }
+  
+  const backupRecovery = await wallet.exportBackupRecovery();
+
+  const wallet2 = new WalletCli(t, "wallet2");
+
+  // Check that the second wallet is a fresh wallet.
+  {
+    const bal = await wallet2.getBalances();
+    t.assertTrue(bal.balances.length === 0);
+  }
+
+  await wallet2.importBackupRecovery({ recovery: backupRecovery });
+
+  await wallet2.runBackupCycle();
+
+  // Check that now the old balance is available!
+  {
+    const bal = await wallet2.getBalances();
+    t.assertTrue(bal.balances.length === 1);
+    console.log(bal);
   }
 }

packages/taler-wallet-core/src/operations/backup/import.ts

@@ -15,68 +15,47 @@
  */
 
 import {
-  Stores,
-  Amounts,
-  CoinSourceType,
-  CoinStatus,
-  RefundState,
   AbortStatus,
-  ProposalStatus,
-  getTimestampNow,
-  encodeCrock,
-  stringToBytes,
-  getRandomBytes,
   AmountJson,
+  Amounts,
   codecForContractTerms,
   CoinSource,
+  CoinSourceType,
+  CoinStatus,
   DenominationStatus,
   DenomSelectionState,
   ExchangeUpdateStatus,
   ExchangeWireInfo,
+  getTimestampNow,
   PayCoinSelection,
   ProposalDownload,
+  ProposalStatus,
   RefreshReason,
   RefreshSessionRecord,
+  RefundState,
   ReserveBankInfo,
   ReserveRecordStatus,
+  Stores,
   TransactionHandle,
   WalletContractData,
   WalletRefundItem,
 } from "../..";
-import { hash } from "../../crypto/primitives/nacl-fast";
 import {
-  WalletBackupContentV1,
-  BackupExchange,
-  BackupCoin,
-  BackupDenomination,
-  BackupReserve,
-  BackupPurchase,
-  BackupProposal,
-  BackupRefreshGroup,
-  BackupBackupProvider,
-  BackupTip,
-  BackupRecoupGroup,
-  BackupWithdrawalGroup,
-  BackupBackupProviderTerms,
-  BackupCoinSource,
   BackupCoinSourceType,
-  BackupExchangeWireFee,
-  BackupRefundItem,
-  BackupRefundState,
-  BackupProposalStatus,
-  BackupRefreshOldCoin,
-  BackupRefreshSession,
   BackupDenomSel,
+  BackupProposalStatus,
+  BackupPurchase,
   BackupRefreshReason,
+  BackupRefundState,
+  WalletBackupContentV1,
 } from "../../types/backupTypes";
-import { canonicalizeBaseUrl, canonicalJson, j2s } from "../../util/helpers";
+import { j2s } from "../../util/helpers";
 import { checkDbInvariant, checkLogicInvariant } from "../../util/invariants";
 import { Logger } from "../../util/logging";
 import { initRetryInfo } from "../../util/retries";
 import { InternalWalletState } from "../state";
 import { provideBackupState } from "./state";
 
-
 const logger = new Logger("operations/backup/import.ts");
 
 function checkBackupInvariant(b: boolean, m?: string): asserts b {
@@ -230,6 +209,9 @@ export async function importBackup(
   cryptoComp: BackupCryptoPrecomputedData,
 ): Promise<void> {
   await provideBackupState(ws);
+
+  logger.info(`importing backup ${j2s(backupBlobArg)}`);
+
   return ws.db.runWithWriteTransaction(
     [
       Stores.config,

packages/taler-wallet-core/src/operations/backup/index.ts

@@ -27,7 +27,11 @@
 import { InternalWalletState } from "../state";
 import { WalletBackupContentV1 } from "../../types/backupTypes";
 import { TransactionHandle } from "../../util/query";
-import { ConfigRecord, Stores } from "../../types/dbTypes";
+import {
+  BackupProviderRecord,
+  ConfigRecord,
+  Stores,
+} from "../../types/dbTypes";
 import { checkDbInvariant, checkLogicInvariant } from "../../util/invariants";
 import { codecForAmountString } from "../../util/amounts";
 import {
@@ -41,7 +45,13 @@ import {
   stringToBytes,
 } from "../../crypto/talerCrypto";
 import { canonicalizeBaseUrl, canonicalJson, j2s } from "../../util/helpers";
-import { getTimestampNow, Timestamp } from "../../util/time";
+import {
+  durationAdd,
+  durationFromSpec,
+  getTimestampNow,
+  Timestamp,
+  timestampAddDuration,
+} from "../../util/time";
 import { URL } from "../../util/url";
 import { AmountString } from "../../types/talerTypes";
 import {
@@ -70,7 +80,7 @@ import {
 } from "../../types/walletTypes";
 import { CryptoApi } from "../../crypto/workers/cryptoApi";
 import { secretbox, secretbox_open } from "../../crypto/primitives/nacl-fast";
-import { confirmPay, preparePayForUri } from "../pay";
+import { checkPaymentByProposalId, confirmPay, preparePayForUri } from "../pay";
 import { exportBackup } from "./export";
 import { BackupCryptoPrecomputedData, importBackup } from "./import";
 import {
@@ -79,6 +89,7 @@ import {
   getWalletBackupState,
   WalletBackupConfState,
 } from "./state";
+import { PaymentStatus } from "../../types/transactionsTypes";
 
 const logger = new Logger("operations/backup.ts");
 
@@ -216,23 +227,31 @@ function deriveBlobSecret(bc: WalletBackupConfState): Uint8Array {
   );
 }
 
+interface BackupForProviderArgs {
+  backupConfig: WalletBackupConfState;
+  provider: BackupProviderRecord;
+  currentBackupHash: ArrayBuffer;
+  encBackup: ArrayBuffer;
+  backupJson: WalletBackupContentV1;
+
   /**
- * Do one backup cycle that consists of:
+   * Should we attempt one more upload after trying
- * 1. Exporting a backup and try to upload it.
+   * to pay?
- *    Stop if this step succeeds.
- * 2. Download, verify and import backups from connected sync accounts.
- * 3. Upload the updated backup blob.
    */
-export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
+  retryAfterPayment: boolean;
-  const providers = await ws.db.iter(Stores.backupProviders).toArray();
+}
-  logger.trace("got backup providers", providers);
-  const backupJson = await exportBackup(ws);
-  const backupConfig = await provideBackupState(ws);
-  const encBackup = await encryptBackup(backupConfig, backupJson);
 
-  const currentBackupHash = hash(encBackup);
+async function runBackupCycleForProvider(
-
+  ws: InternalWalletState,
-  for (const provider of providers) {
+  args: BackupForProviderArgs,
+): Promise<void> {
+  const {
+    backupConfig,
+    provider,
+    currentBackupHash,
+    encBackup,
+    backupJson,
+  } = args;
   const accountKeyPair = deriveAccountKeyPair(backupConfig, provider.baseUrl);
   logger.trace(`trying to upload backup to ${provider.baseUrl}`);
 
@@ -274,35 +293,37 @@ export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
       throw Error("no taler URI available to pay provider");
     }
     const res = await preparePayForUri(ws, talerUri);
-      let proposalId: string | undefined;
+    let proposalId = res.proposalId;
+    let doPay: boolean = false;
     switch (res.status) {
       case PreparePayResultType.InsufficientBalance:
         // FIXME: record in provider state!
         logger.warn("insufficient balance to pay for backup provider");
-          break;
-        case PreparePayResultType.PaymentPossible:
-        case PreparePayResultType.AlreadyConfirmed:
         proposalId = res.proposalId;
         break;
+      case PreparePayResultType.PaymentPossible:
+        doPay = true;
+        break;
+      case PreparePayResultType.AlreadyConfirmed:
+        break;
     }
-      if (!proposalId) {
+
-        continue;
+    // FIXME: check if the provider is overcharging us!
-      }
+
-      const p = proposalId;
     await ws.db.runWithWriteTransaction(
       [Stores.backupProviders],
       async (tx) => {
-          const provRec = await tx.get(
+        const provRec = await tx.get(Stores.backupProviders, provider.baseUrl);
-            Stores.backupProviders,
-            provider.baseUrl,
-          );
         checkDbInvariant(!!provRec);
         const ids = new Set(provRec.paymentProposalIds);
-          ids.add(p);
+        ids.add(proposalId);
-          provRec.paymentProposalIds = Array.from(ids);
+        provRec.paymentProposalIds = Array.from(ids).sort();
+        provRec.currentPaymentProposalId = proposalId;
         await tx.put(Stores.backupProviders, provRec);
       },
     );
+
+    if (doPay) {
       const confirmRes = await confirmPay(ws, proposalId);
       switch (confirmRes.type) {
         case ConfirmPayResultType.Pending:
@@ -310,6 +331,16 @@ export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
           break;
       }
     }
+
+    if (args.retryAfterPayment) {
+      await runBackupCycleForProvider(ws, {
+        ...args,
+        retryAfterPayment: false,
+      });
+    }
+    return;
+  }
+
   if (resp.status === HttpResponseStatus.NoContent) {
     await ws.db.runWithWriteTransaction(
       [Stores.backupProviders],
@@ -320,14 +351,14 @@ export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
         }
         prov.lastBackupHash = encodeCrock(currentBackupHash);
         prov.lastBackupTimestamp = getTimestampNow();
-          prov.lastBackupClock =
+        prov.lastBackupClock = backupJson.clocks[backupJson.current_device_id];
-            backupJson.clocks[backupJson.current_device_id];
         prov.lastError = undefined;
         await tx.put(Stores.backupProviders, prov);
       },
     );
-      continue;
+    return;
   }
+
   if (resp.status === HttpResponseStatus.Conflict) {
     logger.info("conflicting backup found");
     const backupEnc = new Uint8Array(await resp.bytes());
@@ -350,7 +381,7 @@ export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
       },
     );
     logger.info("processed existing backup");
-      continue;
+    return;
   }
 
   // Some other response that we did not expect!
@@ -359,16 +390,41 @@ export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
 
   const err = await readTalerErrorResponse(resp);
   logger.error(`got error response from backup provider: ${j2s(err)}`);
-    await ws.db.runWithWriteTransaction(
+  await ws.db.runWithWriteTransaction([Stores.backupProviders], async (tx) => {
-      [Stores.backupProviders],
-      async (tx) => {
     const prov = await tx.get(Stores.backupProviders, provider.baseUrl);
     if (!prov) {
       return;
     }
     prov.lastError = err;
-      },
+    await tx.put(Stores.backupProviders, prov);
-    );
+  });
+}
+
+/**
+ * Do one backup cycle that consists of:
+ * 1. Exporting a backup and try to upload it.
+ *    Stop if this step succeeds.
+ * 2. Download, verify and import backups from connected sync accounts.
+ * 3. Upload the updated backup blob.
+ */
+export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
+  const providers = await ws.db.iter(Stores.backupProviders).toArray();
+  logger.trace("got backup providers", providers);
+  const backupJson = await exportBackup(ws);
+  const backupConfig = await provideBackupState(ws);
+  const encBackup = await encryptBackup(backupConfig, backupJson);
+
+  const currentBackupHash = hash(encBackup);
+
+  for (const provider of providers) {
+    await runBackupCycleForProvider(ws, {
+      provider,
+      backupJson,
+      backupConfig,
+      encBackup,
+      currentBackupHash,
+      retryAfterPayment: true,
+    });
   }
 }
 
@@ -462,8 +518,15 @@ export interface ProviderInfo {
   lastRemoteClock?: number;
   lastBackupTimestamp?: Timestamp;
   paymentProposalIds: string[];
+  paymentStatus: ProviderPaymentStatus;
 }
 
+export type ProviderPaymentStatus =
+  | ProviderPaymentPaid
+  | ProviderPaymentInsufficientBalance
+  | ProviderPaymentUnpaid
+  | ProviderPaymentPending;
+
 export interface BackupInfo {
   walletRootPub: string;
   deviceId: string;
@@ -483,6 +546,71 @@ export async function importBackupPlain(
   await importBackup(ws, blob, cryptoData);
 }
 
+export enum ProviderPaymentType {
+  Unpaid = "unpaid",
+  Pending = "pending",
+  InsufficientBalance = "insufficient-balance",
+  Paid = "paid",
+}
+
+export interface ProviderPaymentUnpaid {
+  type: ProviderPaymentType.Unpaid;
+}
+
+export interface ProviderPaymentInsufficientBalance {
+  type: ProviderPaymentType.InsufficientBalance;
+}
+
+export interface ProviderPaymentPending {
+  type: ProviderPaymentType.Pending;
+}
+
+export interface ProviderPaymentPaid {
+  type: ProviderPaymentType.Paid;
+  paidUntil: Timestamp;
+}
+
+async function getProviderPaymentInfo(
+  ws: InternalWalletState,
+  provider: BackupProviderRecord,
+): Promise<ProviderPaymentStatus> {
+  if (!provider.currentPaymentProposalId) {
+    return {
+      type: ProviderPaymentType.Unpaid,
+    };
+  }
+  const status = await checkPaymentByProposalId(
+    ws,
+    provider.currentPaymentProposalId,
+  );
+  if (status.status === PreparePayResultType.InsufficientBalance) {
+    return {
+      type: ProviderPaymentType.InsufficientBalance,
+    };
+  }
+  if (status.status === PreparePayResultType.PaymentPossible) {
+    return {
+      type: ProviderPaymentType.Pending,
+    };
+  }
+  if (status.status === PreparePayResultType.AlreadyConfirmed) {
+    if (status.paid) {
+      return {
+        type: ProviderPaymentType.Paid,
+        paidUntil: timestampAddDuration(
+          status.contractTerms.timestamp,
+          durationFromSpec({ years: 1 }),
+        ),
+      };
+    } else {
+      return {
+        type: ProviderPaymentType.Pending,
+      };
+    }
+  }
+  throw Error("not reached");
+}
+
 /**
  * Get information about the current state of wallet backups.
  */
@@ -490,19 +618,24 @@ export async function getBackupInfo(
   ws: InternalWalletState,
 ): Promise<BackupInfo> {
   const backupConfig = await provideBackupState(ws);
-  const providers = await ws.db.iter(Stores.backupProviders).toArray();
+  const providerRecords = await ws.db.iter(Stores.backupProviders).toArray();
-  return {
+  const providers: ProviderInfo[] = [];
-    deviceId: backupConfig.deviceId,
+  for (const x of providerRecords) {
-    lastLocalClock: backupConfig.clocks[backupConfig.deviceId],
+    providers.push({
-    walletRootPub: backupConfig.walletRootPub,
-    providers: providers.map((x) => ({
       active: x.active,
       lastRemoteClock: x.lastBackupClock,
       syncProviderBaseUrl: x.baseUrl,
       lastBackupTimestamp: x.lastBackupTimestamp,
       paymentProposalIds: x.paymentProposalIds,
       lastError: x.lastError,
-    })),
+      paymentStatus: await getProviderPaymentInfo(ws, x),
+    });
+  }
+  return {
+    deviceId: backupConfig.deviceId,
+    lastLocalClock: backupConfig.clocks[backupConfig.deviceId],
+    walletRootPub: backupConfig.walletRootPub,
+    providers,
   };
 }
 

packages/taler-wallet-core/src/operations/pay.ts

@@ -1150,36 +1150,11 @@ async function submitPay(
   };
 }
 
-/**
+export async function checkPaymentByProposalId(
- * Check if a payment for the given taler://pay/ URI is possible.
- *
- * If the payment is possible, the signature are already generated but not
- * yet send to the merchant.
- */
-export async function preparePayForUri(
   ws: InternalWalletState,
-  talerPayUri: string,
+  proposalId: string,
+  sessionId?: string,
 ): Promise<PreparePayResult> {
-  const uriResult = parsePayUri(talerPayUri);
-
-  if (!uriResult) {
-    throw OperationFailedError.fromCode(
-      TalerErrorCode.WALLET_INVALID_TALER_PAY_URI,
-      `invalid taler://pay URI (${talerPayUri})`,
-      {
-        talerPayUri,
-      },
-    );
-  }
-
-  let proposalId = await startDownloadProposal(
-    ws,
-    uriResult.merchantBaseUrl,
-    uriResult.orderId,
-    uriResult.sessionId,
-    uriResult.claimToken,
-  );
-
   let proposal = await ws.db.get(Stores.proposals, proposalId);
   if (!proposal) {
     throw Error(`could not get proposal ${proposalId}`);
@@ -1238,7 +1213,7 @@ export async function preparePayForUri(
     };
   }
 
-  if (purchase.lastSessionId !== uriResult.sessionId) {
+  if (purchase.lastSessionId !== sessionId) {
     logger.trace(
       "automatically re-submitting payment with different session ID",
     );
@@ -1247,7 +1222,7 @@ export async function preparePayForUri(
       if (!p) {
         return;
       }
-      p.lastSessionId = uriResult.sessionId;
+      p.lastSessionId = sessionId;
       await tx.put(Stores.purchases, p);
     });
     const r = await guardOperationException(
@@ -1292,6 +1267,39 @@ export async function preparePayForUri(
   }
 }
 
+/**
+ * Check if a payment for the given taler://pay/ URI is possible.
+ *
+ * If the payment is possible, the signature are already generated but not
+ * yet send to the merchant.
+ */
+export async function preparePayForUri(
+  ws: InternalWalletState,
+  talerPayUri: string,
+): Promise<PreparePayResult> {
+  const uriResult = parsePayUri(talerPayUri);
+
+  if (!uriResult) {
+    throw OperationFailedError.fromCode(
+      TalerErrorCode.WALLET_INVALID_TALER_PAY_URI,
+      `invalid taler://pay URI (${talerPayUri})`,
+      {
+        talerPayUri,
+      },
+    );
+  }
+
+  let proposalId = await startDownloadProposal(
+    ws,
+    uriResult.merchantBaseUrl,
+    uriResult.orderId,
+    uriResult.sessionId,
+    uriResult.claimToken,
+  );
+
+  return checkPaymentByProposalId(ws, proposalId, uriResult.sessionId);
+}
+
 /**
  * Generate deposit permissions for a purchase.
  *

packages/taler-wallet-core/src/types/dbTypes.ts

@@ -1462,8 +1462,19 @@ export interface BackupProviderRecord {
 
   lastBackupTimestamp?: Timestamp;
 
+  /**
+   * Proposal that we're currently trying to pay for.
+   * 
+   * (Also included in paymentProposalIds.)
+   */
   currentPaymentProposalId?: string;
 
+  /**
+   * Proposals that were used to pay (or attempt to pay) the provider.
+   * 
+   * Stored to display a history of payments to the provider, and
+   * to make sure that the wallet isn't overpaying.
+   */
   paymentProposalIds: string[];
 
   /**

packages/taler-wallet-core/src/util/helpers.ts

@@ -59,7 +59,7 @@ export function canonicalizeBaseUrl(url: string): string {
  */
 export function canonicalJson(obj: any): string {
   // Check for cycles, etc.
-  JSON.stringify(obj);
+  obj = JSON.parse(JSON.stringify(obj));
   if (typeof obj === "string" || typeof obj === "number" || obj === null) {
     return JSON.stringify(obj);
   }

packages/taler-wallet-core/src/wallet.ts

@@ -22,7 +22,7 @@
 /**
  * Imports.
  */
-import { TalerErrorCode } from ".";
+import { codecForAny, TalerErrorCode } from ".";
 import { CryptoWorkerFactory } from "./crypto/workers/cryptoApi";
 import {
   addBackupProvider,
@@ -1159,6 +1159,15 @@ export class Wallet {
         await runBackupCycle(this.ws);
         return {};
       }
+      case "exportBackupRecovery": {
+        const resp = await getBackupRecovery(this.ws);
+        return resp;
+      }
+      case "importBackupRecovery": {
+        const req = codecForAny().decode(payload);
+        await loadBackupRecovery(this.ws, req);
+        return {};
+      }
       case "getBackupInfo": {
         const resp = await getBackupInfo(this.ws);
         return resp;