Merge branch 'master' into age-withdraw

packages/taler-harness/src/integrationtests/test-peer-to-peer-push.ts

@@ -89,6 +89,8 @@ export async function runPeerToPeerPushTest(t: GlobalTestState) {
     console.log(resp);
   }
 
+  await w1.walletClient.call(WalletApiOperation.TestingWaitRefreshesFinal, {});
+
   const resp = await w1.walletClient.call(
     WalletApiOperation.InitiatePeerPushDebit,
     {

packages/taler-wallet-core/src/db.ts

@@ -61,6 +61,8 @@ import {
 } from "@gnu-taler/taler-util";
 import {
   DbAccess,
+  DbReadOnlyTransaction,
+  DbReadWriteTransaction,
   describeContents,
   describeIndex,
   describeStore,
@@ -68,6 +70,7 @@ import {
   IndexDescriptor,
   openDatabase,
   StoreDescriptor,
+  StoreNames,
   StoreWithIndexes,
 } from "./util/query.js";
 import { RetryInfo, TaskIdentifiers } from "./operations/common.js";
@@ -2718,6 +2721,15 @@ export const WalletStoresV1 = {
   ),
 };
 
+export type WalletDbReadOnlyTransaction<
+  Stores extends StoreNames<typeof WalletStoresV1> & string,
+> = DbReadOnlyTransaction<typeof WalletStoresV1, Stores>;
+
+export type WalletReadWriteTransaction<
+  Stores extends StoreNames<typeof WalletStoresV1> & string,
+> = DbReadWriteTransaction<typeof WalletStoresV1, Stores>;
+
+
 /**
  * An applied migration.
  */

packages/taler-wallet-core/src/operations/pay-peer-common.ts

@@ -18,27 +18,16 @@
  * Imports.
  */
 import {
-  AgeCommitmentProof,
   AmountJson,
   AmountString,
   Amounts,
   Codec,
-  CoinPublicKeyString,
-  CoinStatus,
-  HttpStatusCode,
   Logger,
-  NotificationType,
-  PayPeerInsufficientBalanceDetails,
-  TalerError,
-  TalerErrorCode,
   TalerProtocolTimestamp,
-  UnblindedSignature,
   buildCodecForObject,
   codecForAmountString,
   codecForTimestamp,
   codecOptional,
-  j2s,
-  strcmp,
 } from "@gnu-taler/taler-util";
 import { SpendCoinDetails } from "../crypto/cryptoImplementation.js";
 import {
@@ -47,10 +36,9 @@ import {
   ReserveRecord,
 } from "../db.js";
 import { InternalWalletState } from "../internal-wallet-state.js";
+import type { SelectedPeerCoin } from "../util/coinSelection.js";
 import { checkDbInvariant } from "../util/invariants.js";
-import { getPeerPaymentBalanceDetailsInTx } from "./balance.js";
 import { getTotalRefreshCost } from "./refresh.js";
-import type { PeerCoinInfo, PeerCoinSelectionRequest, SelectPeerCoinsResult, SelectedPeerCoin } from "../util/coinSelection.js";
 
 const logger = new Logger("operations/peer-to-peer.ts");
 
@@ -96,8 +84,6 @@ export async function queryCoinInfosForSelection(
   return infos;
 }
 
-
-
 export async function getTotalPeerPaymentCost(
   ws: InternalWalletState,
   pcs: SelectedPeerCoin[],

packages/taler-wallet-core/src/operations/pay-peer-pull-credit.ts

@@ -120,6 +120,8 @@ async function queryPurseForPeerPullCredit(
     }
   }
 
+  logger.trace(`purse status: ${j2s(result.response)}`);
+
   const depositTimestamp = result.response.deposit_timestamp;
 
   if (!depositTimestamp || TalerProtocolTimestamp.isNever(depositTimestamp)) {

packages/taler-wallet-core/src/operations/testing.ts

@@ -29,6 +29,7 @@ import {
   TestPayResult,
   TransactionMajorState,
   TransactionMinorState,
+  TransactionType,
   WithdrawTestBalanceRequest,
 } from "@gnu-taler/taler-util";
 import {
@@ -498,6 +499,59 @@ export async function waitUntilDone(ws: InternalWalletState): Promise<void> {
   logger.info("done waiting until all transactions are in a final state");
 }
 
+export async function waitUntilRefreshesDone(
+  ws: InternalWalletState,
+): Promise<void> {
+  logger.info("waiting until all refresh transactions are in a final state");
+  ws.ensureTaskLoopRunning();
+  let p: OpenedPromise<void> | undefined = undefined;
+  const cancelNotifs = ws.addNotificationListener((notif) => {
+    if (!p) {
+      return;
+    }
+    if (notif.type === NotificationType.TransactionStateTransition) {
+      switch (notif.newTxState.major) {
+        case TransactionMajorState.Pending:
+        case TransactionMajorState.Aborting:
+          break;
+        default:
+          p.resolve();
+      }
+    }
+  });
+  while (1) {
+    p = openPromise();
+    const txs = await getTransactions(ws, {
+      includeRefreshes: true,
+      filterByState: "nonfinal",
+    });
+    let finished = true;
+    for (const tx of txs.transactions) {
+      if (tx.type !== TransactionType.Refresh) {
+        continue;
+      }
+      switch (tx.txState.major) {
+        case TransactionMajorState.Pending:
+        case TransactionMajorState.Aborting:
+        case TransactionMajorState.Suspended:
+        case TransactionMajorState.SuspendedAborting:
+          finished = false;
+          logger.info(
+            `continuing waiting, ${tx.transactionId} in ${tx.txState.major}(${tx.txState.minor})`,
+          );
+          break;
+      }
+    }
+    if (finished) {
+      break;
+    }
+    // Wait until transaction state changed
+    await p.promise;
+  }
+  cancelNotifs();
+  logger.info("done waiting until all refreshes are in a final state");
+}
+
 async function waitUntilPendingReady(
   ws: InternalWalletState,
   transactionId: string,

packages/taler-wallet-core/src/util/coinSelection.ts

@@ -29,6 +29,7 @@ import {
   AgeCommitmentProof,
   AgeRestriction,
   AmountJson,
+  AmountLike,
   AmountResponse,
   Amounts,
   AmountString,
@@ -58,7 +59,16 @@ import {
   AllowedExchangeInfo,
   DenominationRecord,
 } from "../db.js";
-import { getExchangeDetails, isWithdrawableDenom } from "../index.js";
+import {
+  DbReadOnlyTransaction,
+  getExchangeDetails,
+  GetReadOnlyAccess,
+  GetReadWriteAccess,
+  isWithdrawableDenom,
+  StoreNames,
+  WalletDbReadOnlyTransaction,
+  WalletStoresV1,
+} from "../index.js";
 import { InternalWalletState } from "../internal-wallet-state.js";
 import {
   getMerchantPaymentBalanceDetails,
@@ -257,10 +267,9 @@ export async function selectPayCoinsNew(
     wireFeeAmortization,
   } = req;
 
-  const [candidateDenoms, wireFeesPerExchange] = await selectPayMerchantCandidates(
+  // FIXME: Why don't we do this in a transaction?
-    ws,
+  const [candidateDenoms, wireFeesPerExchange] =
-    req,
+    await selectPayMerchantCandidates(ws, req);
-  );
 
   const coinPubs: string[] = [];
   const coinContributions: AmountJson[] = [];
@@ -619,7 +628,7 @@ async function selectPayMerchantCandidates(
         if (!accepted) {
           continue;
         }
-        //4.- filter coins restricted by age
+        // 4.- filter coins restricted by age
         let ageLower = 0;
         let ageUpper = AgeRestriction.AGE_UNRESTRICTED;
         if (req.requiredMinimumAge) {
@@ -636,7 +645,7 @@ async function selectPayMerchantCandidates(
               ],
             ),
           );
-        //5.- save denoms with how many coins are available
+        // 5.- save denoms with how many coins are available
         // FIXME: Check that the individual denomination is audited!
         // FIXME: Should we exclude denominations that are
         // not spendable anymore?
@@ -813,7 +822,6 @@ export interface CoinInfo {
   maxAge: number;
 }
 
-
 export interface SelectedPeerCoin {
   coinPub: string;
   coinPriv: string;
@@ -837,33 +845,6 @@ export interface PeerCoinSelectionDetails {
   depositFees: AmountJson;
 }
 
-/**
- * Information about a selected coin for peer to peer payments.
- */
-export interface PeerCoinInfo {
-  /**
-   * Public key of the coin.
-   */
-  coinPub: string;
-
-  coinPriv: string;
-
-  /**
-   * Deposit fee for the coin.
-   */
-  feeDeposit: AmountJson;
-
-  value: AmountJson;
-
-  denomPubHash: string;
-
-  denomSig: UnblindedSignature;
-
-  maxAge: number;
-
-  ageCommitmentProof?: AgeCommitmentProof;
-}
-
 export type SelectPeerCoinsResult =
   | { type: "success"; result: PeerCoinSelectionDetails }
   | {
@@ -887,6 +868,122 @@ export interface PeerCoinSelectionRequest {
   repair?: PeerCoinRepair;
 }
 
+/**
+ * Get coin availability information for a certain exchange.
+ */
+async function selectPayPeerCandidatesForExchange(
+  ws: InternalWalletState,
+  tx: WalletDbReadOnlyTransaction<"coinAvailability" | "denominations">,
+  exchangeBaseUrl: string,
+): Promise<AvailableDenom[]> {
+  const denoms: AvailableDenom[] = [];
+
+  let ageLower = 0;
+  let ageUpper = AgeRestriction.AGE_UNRESTRICTED;
+  const myExchangeCoins =
+    await tx.coinAvailability.indexes.byExchangeAgeAvailability.getAll(
+      GlobalIDB.KeyRange.bound(
+        [exchangeBaseUrl, ageLower, 1],
+        [exchangeBaseUrl, ageUpper, Number.MAX_SAFE_INTEGER],
+      ),
+    );
+
+  for (const coinAvail of myExchangeCoins) {
+    const denom = await tx.denominations.get([
+      coinAvail.exchangeBaseUrl,
+      coinAvail.denomPubHash,
+    ]);
+    checkDbInvariant(!!denom);
+    if (denom.isRevoked || !denom.isOffered) {
+      continue;
+    }
+    denoms.push({
+      ...DenominationRecord.toDenomInfo(denom),
+      numAvailable: coinAvail.freshCoinCount ?? 0,
+      maxAge: coinAvail.maxAge,
+    });
+  }
+  // Sort by available amount (descending),  deposit fee (ascending) and
+  // denomPub (ascending) if deposit fee is the same
+  // (to guarantee deterministic results)
+  denoms.sort(
+    (o1, o2) =>
+      -Amounts.cmp(o1.value, o2.value) ||
+      Amounts.cmp(o1.feeDeposit, o2.feeDeposit) ||
+      strcmp(o1.denomPubHash, o2.denomPubHash),
+  );
+
+  return denoms;
+}
+
+interface PeerCoinSelectionTally {
+  amountAcc: AmountJson;
+  depositFeesAcc: AmountJson;
+  lastDepositFee: AmountJson;
+}
+
+function greedySelectPeer(
+  candidates: AvailableDenom[],
+  instructedAmount: AmountLike,
+  tally: PeerCoinSelectionTally,
+): SelResult | undefined {
+  const selectedDenom: SelResult = {};
+  for (const denom of candidates) {
+    const contributions: AmountJson[] = [];
+    for (
+      let i = 0;
+      i < denom.numAvailable &&
+      Amounts.cmp(tally.amountAcc, instructedAmount) < 0;
+      i++
+    ) {
+      const amountPayRemaining = Amounts.sub(
+        instructedAmount,
+        tally.amountAcc,
+      ).amount;
+      const coinSpend = Amounts.max(
+        Amounts.min(amountPayRemaining, denom.value),
+        denom.feeDeposit,
+      );
+      tally.amountAcc = Amounts.add(tally.amountAcc, coinSpend).amount;
+      // Since this is a peer payment, there is no merchant to
+      // potentially cover the deposit fees.
+      tally.amountAcc = Amounts.sub(tally.amountAcc, denom.feeDeposit).amount;
+      tally.depositFeesAcc = Amounts.add(
+        tally.depositFeesAcc,
+        denom.feeDeposit,
+      ).amount;
+      tally.lastDepositFee = Amounts.parseOrThrow(denom.feeDeposit);
+      contributions.push(coinSpend);
+    }
+    if (contributions.length > 0) {
+      const avKey = makeAvailabilityKey(
+        denom.exchangeBaseUrl,
+        denom.denomPubHash,
+        denom.maxAge,
+      );
+      let sd = selectedDenom[avKey];
+      if (!sd) {
+        sd = {
+          contributions: [],
+          denomPubHash: denom.denomPubHash,
+          exchangeBaseUrl: denom.exchangeBaseUrl,
+          maxAge: denom.maxAge,
+        };
+      }
+      sd.contributions.push(...contributions);
+      selectedDenom[avKey] = sd;
+    }
+    if (Amounts.cmp(tally.amountAcc, instructedAmount) >= 0) {
+      break;
+    }
+  }
+
+  if (Amounts.cmp(tally.amountAcc, instructedAmount) >= 0) {
+    return selectedDenom;
+  }
+  return undefined;
+}
+
 export async function selectPeerCoins(
   ws: InternalWalletState,
   req: PeerCoinSelectionRequest,
@@ -915,42 +1012,16 @@ export async function selectPeerCoins(
         if (exch.detailsPointer?.currency !== currency) {
           continue;
         }
-        // FIXME: Can't we do this faster by using coinAvailability?
+        const candidates = await selectPayPeerCandidatesForExchange(
-        const coins = (
+          ws,
-          await tx.coins.indexes.byBaseUrl.getAll(exch.baseUrl)
+          tx,
-        ).filter((x) => x.status === CoinStatus.Fresh);
+          exch.baseUrl,
-        const coinInfos: PeerCoinInfo[] = [];
-        for (const coin of coins) {
-          const denom = await ws.getDenomInfo(
-            ws,
-            tx,
-            coin.exchangeBaseUrl,
-            coin.denomPubHash,
-          );
-          if (!denom) {
-            throw Error("denom not found");
-          }
-          coinInfos.push({
-            coinPub: coin.coinPub,
-            feeDeposit: Amounts.parseOrThrow(denom.feeDeposit),
-            value: Amounts.parseOrThrow(denom.value),
-            denomPubHash: denom.denomPubHash,
-            coinPriv: coin.coinPriv,
-            denomSig: coin.denomSig,
-            maxAge: coin.maxAge,
-            ageCommitmentProof: coin.ageCommitmentProof,
-          });
-        }
-        if (coinInfos.length === 0) {
-          continue;
-        }
-        coinInfos.sort(
-          (o1, o2) =>
-            -Amounts.cmp(o1.value, o2.value) ||
-            strcmp(o1.denomPubHash, o2.denomPubHash),
         );
-        let amountAcc = Amounts.zeroOfCurrency(currency);
+        const tally: PeerCoinSelectionTally = {
-        let depositFeesAcc = Amounts.zeroOfCurrency(currency);
+          amountAcc: Amounts.zeroOfCurrency(currency),
+          depositFeesAcc: Amounts.zeroOfCurrency(currency),
+          lastDepositFee: Amounts.zeroOfCurrency(currency),
+        };
         const resCoins: {
           coinPub: string;
           coinPriv: string;
@@ -959,9 +1030,8 @@ export async function selectPeerCoins(
           denomSig: UnblindedSignature;
           ageCommitmentProof: AgeCommitmentProof | undefined;
         }[] = [];
-        let lastDepositFee = Amounts.zeroOfCurrency(currency);
 
-        if (req.repair) {
+        if (req.repair && req.repair.exchangeBaseUrl === exch.baseUrl) {
           for (let i = 0; i < req.repair.coinPubs.length; i++) {
             const contrib = req.repair.contribs[i];
             const coin = await tx.coins.get(req.repair.coinPubs[i]);
@@ -984,49 +1054,70 @@ export async function selectPeerCoins(
               ageCommitmentProof: coin.ageCommitmentProof,
             });
             const depositFee = Amounts.parseOrThrow(denom.feeDeposit);
-            lastDepositFee = depositFee;
+            tally.lastDepositFee = depositFee;
-            amountAcc = Amounts.add(
+            tally.amountAcc = Amounts.add(
-              amountAcc,
+              tally.amountAcc,
               Amounts.sub(contrib, depositFee).amount,
             ).amount;
-            depositFeesAcc = Amounts.add(depositFeesAcc, depositFee).amount;
+            tally.depositFeesAcc = Amounts.add(
+              tally.depositFeesAcc,
+              depositFee,
+            ).amount;
           }
         }
 
-        for (const coin of coinInfos) {
+        const selectedDenom = greedySelectPeer(
-          if (Amounts.cmp(amountAcc, instructedAmount) >= 0) {
+          candidates,
-            break;
+          instructedAmount,
+          tally,
+        );
+
+        if (selectedDenom) {
+          for (const dph of Object.keys(selectedDenom)) {
+            const selInfo = selectedDenom[dph];
+            const numRequested = selInfo.contributions.length;
+            const query = [
+              selInfo.exchangeBaseUrl,
+              selInfo.denomPubHash,
+              selInfo.maxAge,
+              CoinStatus.Fresh,
+            ];
+            logger.info(`query: ${j2s(query)}`);
+            const coins =
+              await tx.coins.indexes.byExchangeDenomPubHashAndAgeAndStatus.getAll(
+                query,
+                numRequested,
+              );
+            if (coins.length != numRequested) {
+              throw Error(
+                `coin selection failed (not available anymore, got only ${coins.length}/${numRequested})`,
+              );
+            }
+            for (let i = 0; i < selInfo.contributions.length; i++) {
+              resCoins.push({
+                coinPriv: coins[i].coinPriv,
+                coinPub: coins[i].coinPub,
+                contribution: Amounts.stringify(selInfo.contributions[i]),
+                ageCommitmentProof: coins[i].ageCommitmentProof,
+                denomPubHash: selInfo.denomPubHash,
+                denomSig: coins[i].denomSig,
+              });
+            }
           }
-          const gap = Amounts.add(
+
-            coin.feeDeposit,
-            Amounts.sub(instructedAmount, amountAcc).amount,
-          ).amount;
-          const contrib = Amounts.min(gap, coin.value);
-          amountAcc = Amounts.add(
-            amountAcc,
-            Amounts.sub(contrib, coin.feeDeposit).amount,
-          ).amount;
-          depositFeesAcc = Amounts.add(depositFeesAcc, coin.feeDeposit).amount;
-          resCoins.push({
-            coinPriv: coin.coinPriv,
-            coinPub: coin.coinPub,
-            contribution: Amounts.stringify(contrib),
-            denomPubHash: coin.denomPubHash,
-            denomSig: coin.denomSig,
-            ageCommitmentProof: coin.ageCommitmentProof,
-          });
-          lastDepositFee = coin.feeDeposit;
-        }
-        if (Amounts.cmp(amountAcc, instructedAmount) >= 0) {
           const res: PeerCoinSelectionDetails = {
             exchangeBaseUrl: exch.baseUrl,
             coins: resCoins,
-            depositFees: depositFeesAcc,
+            depositFees: tally.depositFeesAcc,
           };
           return { type: "success", result: res };
         }
-        const diff = Amounts.sub(instructedAmount, amountAcc).amount;
+
-        exchangeFeeGap[exch.baseUrl] = Amounts.add(lastDepositFee, diff).amount;
+        const diff = Amounts.sub(instructedAmount, tally.amountAcc).amount;
+        exchangeFeeGap[exch.baseUrl] = Amounts.add(
+          tally.lastDepositFee,
+          diff,
+        ).amount;
 
         continue;
       }

packages/taler-wallet-core/src/util/query.ts

@@ -429,6 +429,46 @@ export type GetReadOnlyAccess<BoundStores> = {
     : unknown;
 };
 
+export type StoreNames<StoreMap> = StoreMap extends {
+  [P in keyof StoreMap]: StoreWithIndexes<infer SN1, infer SD1, infer IM1>;
+}
+  ? keyof StoreMap
+  : unknown;
+
+export type DbReadOnlyTransaction<
+  StoreMap,
+  Stores extends StoreNames<StoreMap> & string,
+> = StoreMap extends {
+  [P in Stores]: StoreWithIndexes<infer SN1, infer SD1, infer IM1>;
+}
+  ? {
+      [P in Stores]: StoreMap[P] extends StoreWithIndexes<
+        infer SN,
+        infer SD,
+        infer IM
+      >
+        ? StoreReadOnlyAccessor<GetRecordType<SD>, IM>
+        : unknown;
+    }
+  : unknown;
+
+export type DbReadWriteTransaction<
+  StoreMap,
+  Stores extends StoreNames<StoreMap> & string,
+> = StoreMap extends {
+  [P in Stores]: StoreWithIndexes<infer SN1, infer SD1, infer IM1>;
+}
+  ? {
+      [P in Stores]: StoreMap[P] extends StoreWithIndexes<
+        infer SN,
+        infer SD,
+        infer IM
+      >
+        ? StoreReadWriteAccessor<GetRecordType<SD>, IM>
+        : unknown;
+    }
+  : unknown;
+
 export type GetReadWriteAccess<BoundStores> = {
   [P in keyof BoundStores]: BoundStores[P] extends StoreWithIndexes<
     infer SN,

packages/taler-wallet-core/src/wallet-api-types.ts

@@ -212,6 +212,7 @@ export enum WalletApiOperation {
   ApplyDevExperiment = "applyDevExperiment",
   ValidateIban = "validateIban",
   TestingWaitTransactionsFinal = "testingWaitTransactionsFinal",
+  TestingWaitRefreshesFinal = "testingWaitRefreshesFinal",
   GetScopedCurrencyInfo = "getScopedCurrencyInfo",
 }
 
@@ -976,6 +977,15 @@ export type TestingWaitTransactionsFinal = {
   response: EmptyObject;
 };
 
+/**
+ * Wait until all refresh transactions are in a final state.
+ */
+export type TestingWaitRefreshesFinal = {
+  op: WalletApiOperation.TestingWaitRefreshesFinal;
+  request: EmptyObject;
+  response: EmptyObject;
+};
+
 /**
  * Set a coin as (un-)suspended.
  * Suspended coins won't be used for payments.
@@ -1080,6 +1090,7 @@ export type WalletOperations = {
   [WalletApiOperation.ApplyDevExperiment]: ApplyDevExperimentOp;
   [WalletApiOperation.ValidateIban]: ValidateIbanOp;
   [WalletApiOperation.TestingWaitTransactionsFinal]: TestingWaitTransactionsFinal;
+  [WalletApiOperation.TestingWaitRefreshesFinal]: TestingWaitRefreshesFinal;
   [WalletApiOperation.GetScopedCurrencyInfo]: GetScopedCurrencyInfoOp;
 };
 

packages/taler-wallet-core/src/wallet.ts

@@ -247,6 +247,7 @@ import {
   runIntegrationTest2,
   testPay,
   waitUntilDone,
+  waitUntilRefreshesDone,
   withdrawTestBalance,
 } from "./operations/testing.js";
 import {
@@ -1586,6 +1587,8 @@ async function dispatchRequestInternal<Op extends WalletApiOperation>(
     }
     case WalletApiOperation.TestingWaitTransactionsFinal:
       return await waitUntilDone(ws);
+    case WalletApiOperation.TestingWaitRefreshesFinal:
+      return await waitUntilRefreshesDone(ws);
     // default:
     //  assertUnreachable(operation);
   }
@@ -1685,7 +1688,8 @@ export class Wallet {
 
   public static defaultConfig: Readonly<WalletConfig> = {
     builtin: {
-      exchanges: ["https://exchange.demo.taler.net/"],
+      //exchanges: ["https://exchange.demo.taler.net/"],
+      exchanges: [],
       auditors: [
         {
           currency: "KUDOS",