throttling / allow non-json requests

src/android/index.ts

@@ -26,7 +26,7 @@ import {
 } from "../headless/helpers";
 import { openPromise, OpenedPromise } from "../util/promiseUtils";
 import fs = require("fs");
-import { HttpRequestLibrary, HttpResponse } from "../util/http";
+import { HttpRequestLibrary, HttpResponse, HttpRequestOptions } from "../util/http";
 
 // @ts-ignore: special built-in module
 //import akono = require("akono");
@@ -44,7 +44,7 @@ export class AndroidHttpLib implements HttpRequestLibrary {
 
   constructor(private sendMessage: (m: string) => void) {}
 
-  get(url: string): Promise<HttpResponse> {
+  get(url: string, opt?: HttpRequestOptions): Promise<HttpResponse> {
     if (this.useNfcTunnel) {
       const myId = this.requestId++;
       const p = openPromise<HttpResponse>();
@@ -62,11 +62,11 @@ export class AndroidHttpLib implements HttpRequestLibrary {
       );
       return p.promise;
     } else {
-      return this.nodeHttpLib.get(url);
+      return this.nodeHttpLib.get(url, opt);
     }
   }
 
-  postJson(url: string, body: any): Promise<import("../util/http").HttpResponse> {
+  postJson(url: string, body: any, opt?: HttpRequestOptions): Promise<import("../util/http").HttpResponse> {
     if (this.useNfcTunnel) {
       const myId = this.requestId++;
       const p = openPromise<HttpResponse>();
@@ -81,7 +81,7 @@ export class AndroidHttpLib implements HttpRequestLibrary {
       );
       return p.promise;
     } else {
-      return this.nodeHttpLib.postJson(url, body);
+      return this.nodeHttpLib.postJson(url, body, opt);
     }
   }
 
@@ -91,8 +91,14 @@ export class AndroidHttpLib implements HttpRequestLibrary {
     if (!p) {
       console.error(`no matching request for tunneled HTTP response, id=${myId}`);
     }
-    if (msg.status == 200) {
+    if (msg.status != 0) {
-      p.resolve({ responseJson: msg.responseJson, status: msg.status });
+      const resp: HttpResponse = {
+        headers: {},
+        status: msg.status,
+        json: async () => JSON.parse(msg.responseText),
+        text: async () => msg.responseText,
+      };
+      p.resolve(resp);
     } else {
       p.reject(new Error(`unexpected HTTP status code ${msg.status}`));
     }

src/headless/helpers.ts

@@ -24,8 +24,11 @@
 import { Wallet } from "../wallet";
 import { MemoryBackend, BridgeIDBFactory, shimIndexedDB } from "idb-bridge";
 import { openTalerDb } from "../db";
-import Axios from "axios";
+import Axios, { AxiosPromise, AxiosResponse } from "axios";
-import { HttpRequestLibrary } from "../util/http";
+import {
+  HttpRequestLibrary,
+  HttpRequestOptions,
+} from "../util/http";
 import * as amounts from "../util/amounts";
 import { Bank } from "./bank";
 
@@ -34,45 +37,73 @@ import { Logger } from "../util/logging";
 import { NodeThreadCryptoWorkerFactory } from "../crypto/workers/nodeThreadWorker";
 import { NotificationType, WalletNotification } from "../walletTypes";
 import { SynchronousCryptoWorkerFactory } from "../crypto/workers/synchronousWorker";
+import { RequestThrottler } from "../util/RequestThrottler";
 
 const logger = new Logger("helpers.ts");
 
-
 export class NodeHttpLib implements HttpRequestLibrary {
-  async get(url: string): Promise<import("../util/http").HttpResponse> {
+  private throttle = new RequestThrottler();
+
+  private async req(
+    method: "post" | "get",
+    url: string,
+    body: any,
+    opt?: HttpRequestOptions,
+  ) {
+    if (this.throttle.applyThrottle(url)) {
+      throw Error("request throttled");
+    }
+    let resp: AxiosResponse;
     try {
-      const resp = await Axios({
+      resp = await Axios({
-        method: "get",
+        method,
         url: url,
-        responseType: "json",
+        responseType: "text",
+        headers: opt?.headers,
+        validateStatus: () => true,
+        transformResponse: (x) => x,
+        data: body,
       });
-      return {
-        responseJson: resp.data,
-        status: resp.status,
-      };
     } catch (e) {
       throw e;
     }
+    const respText = resp.data;
+    if (typeof respText !== "string") {
+      throw Error("unexpected response type");
+    }
+    const makeJson = async () => {
+      let responseJson;
+      try {
+        responseJson = JSON.parse(respText);
+      } catch (e) {
+        throw Error("Invalid JSON from HTTP response");
+      }
+      if (responseJson === null || typeof responseJson !== "object") {
+        throw Error("Invalid JSON from HTTP response");
+      }
+      return responseJson;
+    };
+    return {
+      headers: resp.headers,
+      status: resp.status,
+      text: async () => resp.data,
+      json: makeJson,
+    };
+  }
+
+  async get(
+    url: string,
+    opt?: HttpRequestOptions,
+  ): Promise<import("../util/http").HttpResponse> {
+    return this.req("get", url, undefined, opt);
   }
 
   async postJson(
     url: string,
     body: any,
+    opt?: HttpRequestOptions,
   ): Promise<import("../util/http").HttpResponse> {
-    try {
+    return this.req("post", url, body, opt);
-      const resp = await Axios({
-        method: "post",
-        url: url,
-        responseType: "json",
-        data: body,
-      });
-      return {
-        responseJson: resp.data,
-        status: resp.status,
-      };
-    } catch (e) {
-      throw e;
-    }
   }
 }
 
@@ -103,8 +134,6 @@ export interface DefaultNodeWalletArgs {
 export async function getDefaultNodeWallet(
   args: DefaultNodeWalletArgs = {},
 ): Promise<Wallet> {
-
-
   BridgeIDBFactory.enableTracing = false;
   const myBackend = new MemoryBackend();
   myBackend.enableTracing = false;
@@ -112,7 +141,9 @@ export async function getDefaultNodeWallet(
   const storagePath = args.persistentStoragePath;
   if (storagePath) {
     try {
-      const dbContentStr: string = fs.readFileSync(storagePath, { encoding: "utf-8" });
+      const dbContentStr: string = fs.readFileSync(storagePath, {
+        encoding: "utf-8",
+      });
       const dbContent = JSON.parse(dbContentStr);
       myBackend.importDump(dbContent);
     } catch (e) {
@@ -125,7 +156,9 @@ export async function getDefaultNodeWallet(
         return;
       }
       const dbContent = myBackend.exportDump();
-      fs.writeFileSync(storagePath, JSON.stringify(dbContent, undefined, 2), { encoding: "utf-8" });
+      fs.writeFileSync(storagePath, JSON.stringify(dbContent, undefined, 2), {
+        encoding: "utf-8",
+      });
     };
   }
 
@@ -164,11 +197,7 @@ export async function getDefaultNodeWallet(
 
   const worker = new NodeThreadCryptoWorkerFactory();
 
-  const w = new Wallet(
+  const w = new Wallet(myDb, myHttpLib, worker);
-    myDb,
-    myHttpLib,
-    worker,
-  );
   if (args.notifyHandler) {
     w.addNotificationListener(args.notifyHandler);
   }
@@ -193,27 +222,24 @@ export async function withdrawTestBalance(
 
   const bankUser = await bank.registerRandomUser();
 
-  logger.trace(`Registered bank user ${JSON.stringify(bankUser)}`)
+  logger.trace(`Registered bank user ${JSON.stringify(bankUser)}`);
 
-  const exchangePaytoUri = await myWallet.getExchangePaytoUri(
+  const exchangePaytoUri = await myWallet.getExchangePaytoUri(exchangeBaseUrl, [
-    exchangeBaseUrl,
+    "x-taler-bank",
-    ["x-taler-bank"],
+  ]);
-  );
 
   const donePromise = new Promise((resolve, reject) => {
-    myWallet.addNotificationListener((n) => {
+    myWallet.addNotificationListener(n => {
-      if (n.type === NotificationType.ReserveDepleted && n.reservePub === reservePub ) {
+      if (
+        n.type === NotificationType.ReserveDepleted &&
+        n.reservePub === reservePub
+      ) {
         resolve();
       }
     });
   });
 
-  await bank.createReserve(
+  await bank.createReserve(bankUser, amount, reservePub, exchangePaytoUri);
-    bankUser,
-    amount,
-    reservePub,
-    exchangePaytoUri,
-  );
 
   await myWallet.confirmReserve({ reservePub: reserveResponse.reservePub });
   await donePromise;

src/util/http.ts

@@ -24,16 +24,25 @@
  */
 export interface HttpResponse {
   status: number;
-  responseJson: object & any;
+  headers: { [name: string]: string };
+  json(): Promise<any>;
+  text(): Promise<string>;
+}
+
+export interface HttpRequestOptions {
+  headers?: { [name: string]: string };
 }
 
 /**
- * The request library is bundled into an interface to make mocking easy.
+ * The request library is bundled into an interface to m  responseJson: object & any;ake mocking easy.
  */
 export interface HttpRequestLibrary {
-  get(url: string): Promise<HttpResponse>;
+  get(url: string, opt?: HttpRequestOptions): Promise<HttpResponse>;
-
+  postJson(
-  postJson(url: string, body: any): Promise<HttpResponse>;
+    url: string,
+    body: any,
+    opt?: HttpRequestOptions,
+  ): Promise<HttpResponse>;
 }
 
 /**
@@ -44,13 +53,20 @@ export class BrowserHttpLib implements HttpRequestLibrary {
   private req(
     method: string,
     url: string,
-    options?: any,
+    requestBody?: any,
+    options?: HttpRequestOptions,
   ): Promise<HttpResponse> {
     return new Promise<HttpResponse>((resolve, reject) => {
       const myRequest = new XMLHttpRequest();
       myRequest.open(method, url);
-      if (options && options.req) {
+      if (options?.headers) {
-        myRequest.send(options.req);
+        for (const headerName in options.headers) {
+          myRequest.setRequestHeader(headerName, options.headers[headerName]);
+        }
+      }
+      myRequest.setRequestHeader;
+      if (requestBody) {
+        myRequest.send(requestBody);
       } else {
         myRequest.send();
       }
@@ -63,31 +79,42 @@ export class BrowserHttpLib implements HttpRequestLibrary {
       myRequest.addEventListener("readystatechange", e => {
         if (myRequest.readyState === XMLHttpRequest.DONE) {
           if (myRequest.status === 0) {
-            reject(Error("HTTP Request failed (status code 0, maybe URI scheme is wrong?)"))
-            return;
-          }
-          if (myRequest.status != 200) {
             reject(
               Error(
-                `HTTP Response with unexpected status code ${myRequest.status}: ${myRequest.statusText}`,
+                "HTTP Request failed (status code 0, maybe URI scheme is wrong?)",
               ),
             );
             return;
           }
-          let responseJson;
+          const makeJson = async () => {
-          try {
+            let responseJson;
-            responseJson = JSON.parse(myRequest.responseText);
+            try {
-          } catch (e) {
+              responseJson = JSON.parse(myRequest.responseText);
-            reject(Error("Invalid JSON from HTTP response"));
+            } catch (e) {
-            return;
+              throw Error("Invalid JSON from HTTP response");
-          }
+            }
-          if (responseJson === null || typeof responseJson !== "object") {
+            if (responseJson === null || typeof responseJson !== "object") {
-            reject(Error("Invalid JSON from HTTP response"));
+              throw Error("Invalid JSON from HTTP response");
-            return;
+            }
-          }
+            return responseJson;
-          const resp = {
+          };
-            responseJson: responseJson,
+
+          const headers = myRequest.getAllResponseHeaders();
+          const arr = headers.trim().split(/[\r\n]+/);
+
+          // Create a map of header names to values
+          const headerMap: { [name: string]: string } = {};
+          arr.forEach(function(line) {
+            const parts = line.split(": ");
+            const header = parts.shift();
+            const value = parts.join(": ");
+            headerMap[header!] = value;
+          });
+          const resp: HttpResponse = {
             status: myRequest.status,
+            headers: headerMap,
+            json: makeJson,
+            text: async () => myRequest.responseText,
           };
           resolve(resp);
         }
@@ -95,15 +122,15 @@ export class BrowserHttpLib implements HttpRequestLibrary {
     });
   }
 
-  get(url: string) {
+  get(url: string, opt?: HttpRequestOptions) {
-    return this.req("get", url);
+    return this.req("get", url, undefined, opt);
   }
 
-  postJson(url: string, body: any) {
+  postJson(url: string, body: any, opt?: HttpRequestOptions) {
-    return this.req("post", url, { req: JSON.stringify(body) });
+    return this.req("post", url, JSON.stringify(body), opt);
   }
 
-  postForm(url: string, form: any) {
+  stop() {
-    return this.req("post", url, { req: form });
+    // Nothing to do
   }
 }

src/wallet-impl/exchanges.ts

@@ -112,7 +112,11 @@ async function updateExchangeWithKeys(
 
   let keysResp;
   try {
-    keysResp = await ws.http.get(keysUrl.href);
+    const r = await ws.http.get(keysUrl.href);
+    if (r.status !== 200) {
+      throw Error(`unexpected status for keys: ${r.status}`);
+    }
+    keysResp = await r.json();
   } catch (e) {
     const m = `Fetching keys failed: ${e.message}`;
     await setExchangeError(ws, baseUrl, {
@@ -126,7 +130,7 @@ async function updateExchangeWithKeys(
   }
   let exchangeKeysJson: KeysJson;
   try {
-    exchangeKeysJson = KeysJson.checked(keysResp.responseJson);
+    exchangeKeysJson = KeysJson.checked(keysResp);
   } catch (e) {
     const m = `Parsing /keys response failed: ${e.message}`;
     await setExchangeError(ws, baseUrl, {
@@ -242,8 +246,10 @@ async function updateExchangeWithWireInfo(
   reqUrl.searchParams.set("cacheBreaker", WALLET_CACHE_BREAKER_CLIENT_VERSION);
 
   const resp = await ws.http.get(reqUrl.href);
-
+  if (resp.status !== 200) {
-  const wiJson = resp.responseJson;
+    throw Error(`/wire response has unexpected status code (${resp.status})`);
+  }
+  const wiJson = await resp.json();
   if (!wiJson) {
     throw Error("/wire response malformed");
   }

src/wallet-impl/pay.ts

@@ -441,7 +441,11 @@ export async function abortFailedPayment(
     throw e;
   }
 
-  const refundResponse = MerchantRefundResponse.checked(resp.responseJson);
+  if (resp.status !== 200) {
+    throw Error(`unexpected status for /pay (${resp.status})`);
+  }
+
+  const refundResponse = MerchantRefundResponse.checked(await resp.json());
   await acceptRefundResponse(ws, purchase.proposalId, refundResponse);
 
   await runWithWriteTransaction(ws.db, [Stores.purchases], async tx => {
@@ -597,7 +601,11 @@ async function processDownloadProposalImpl(
     throw e;
   }
 
-  const proposalResp = Proposal.checked(resp.responseJson);
+  if (resp.status !== 200) {
+    throw Error(`contract download failed with status ${resp.status}`);
+  }
+
+  const proposalResp = Proposal.checked(await resp.json());
 
   const contractTermsHash = await ws.cryptoApi.hashString(
     canonicalJson(proposalResp.contract_terms),
@@ -717,7 +725,10 @@ export async function submitPay(
     console.log("payment failed", e);
     throw e;
   }
-  const merchantResp = resp.responseJson;
+  if (resp.status !== 200) {
+    throw Error(`unexpected status (${resp.status}) for /pay`);
+  }
+  const merchantResp = await resp.json();
   console.log("got success from pay URL");
 
   const merchantPub = purchase.contractTerms.merchant_pub;
@@ -1317,8 +1328,11 @@ async function processPurchaseQueryRefundImpl(
     console.error("error downloading refund permission", e);
     throw e;
   }
+  if (resp.status !== 200) {
+    throw Error(`unexpected status code (${resp.status}) for /refund`);
+  }
 
-  const refundResponse = MerchantRefundResponse.checked(resp.responseJson);
+  const refundResponse = MerchantRefundResponse.checked(await resp.json());
   await acceptRefundResponse(ws, proposalId, refundResponse);
 }
 

src/wallet-impl/payback.ts

@@ -76,7 +76,7 @@ export async function payback(
   if (resp.status !== 200) {
     throw Error();
   }
-  const paybackConfirmation = PaybackConfirmation.checked(resp.responseJson);
+  const paybackConfirmation = PaybackConfirmation.checked(await resp.json());
   if (paybackConfirmation.reserve_pub !== coin.reservePub) {
     throw Error(`Coin's reserve doesn't match reserve on payback`);
   }

src/wallet-impl/pending.ts

@@ -238,7 +238,7 @@ async function gatherCoinsPending(
   // Refreshing dirty coins is always due.
   await tx.iter(Stores.coins).forEach(coin => {
     if (coin.status == CoinStatus.Dirty) {
-      resp.nextRetryDelay.d_ms = 0;
+      resp.nextRetryDelay = { d_ms: 0 };
       resp.pendingOperations.push({
         givesLifeness: true,
         type: "dirty-coin",

src/wallet-impl/refresh.ts

@@ -118,15 +118,18 @@ async function refreshMelt(
   };
   logger.trace("melt request:", meltReq);
   const resp = await ws.http.postJson(reqUrl.href, meltReq);
-
-  logger.trace("melt response:", resp.responseJson);
-
   if (resp.status !== 200) {
-    console.error(resp.responseJson);
+    throw Error(`unexpected status code ${resp.status} for refresh/melt`);
-    throw Error("refresh failed");
   }
 
-  const respJson = resp.responseJson;
+  const respJson = await resp.json();
+
+  logger.trace("melt response:", respJson);
+
+  if (resp.status !== 200) {
+    console.error(respJson);
+    throw Error("refresh failed");
+  }
 
   const norevealIndex = respJson.noreveal_index;
 
@@ -228,7 +231,7 @@ async function refreshReveal(
     return;
   }
 
-  const respJson = resp.responseJson;
+  const respJson = await resp.json();
 
   if (!respJson.ev_sigs || !Array.isArray(respJson.ev_sigs)) {
     console.error("/refresh/reveal did not contain ev_sigs");

src/wallet-impl/reserves.ts

@@ -282,7 +282,10 @@ async function processReserveBankStatusImpl(
   let status: WithdrawOperationStatusResponse;
   try {
     const statusResp = await ws.http.get(bankStatusUrl);
-    status = WithdrawOperationStatusResponse.checked(statusResp.responseJson);
+    if (statusResp.status !== 200) {
+      throw Error(`unexpected status ${statusResp.status} for bank status query`);
+    }
+    status = WithdrawOperationStatusResponse.checked(await statusResp.json());
   } catch (e) {
     throw e;
   }
@@ -378,22 +381,24 @@ async function updateReserve(
   let resp;
   try {
     resp = await ws.http.get(reqUrl.href);
-  } catch (e) {
+    if (resp.status === 404) {
-    if (e.response?.status === 404) {
       const m = "The exchange does not know about this reserve (yet).";
       await incrementReserveRetry(ws, reservePub, undefined);
       return;
-    } else {
-      const m = e.message;
-      await incrementReserveRetry(ws, reservePub, {
-        type: "network",
-        details: {},
-        message: m,
-      });
-      throw new OperationFailedAndReportedError(m);
     }
+    if (resp.status !== 200) {
+      throw Error(`unexpected status code ${resp.status} for reserve/status`)
+    }
+  } catch (e) {
+    const m = e.message;
+    await incrementReserveRetry(ws, reservePub, {
+      type: "network",
+      details: {},
+      message: m,
+    });
+    throw new OperationFailedAndReportedError(m);
   }
-  const reserveInfo = ReserveStatus.checked(resp.responseJson);
+  const reserveInfo = ReserveStatus.checked(await resp.json());
   const balance = Amounts.parseOrThrow(reserveInfo.balance);
   await oneShotMutate(ws.db, Stores.reserves, reserve.reservePub, r => {
     if (r.reserveStatus !== ReserveRecordStatus.QUERYING_STATUS) {

src/wallet-impl/return.ts

@@ -238,7 +238,7 @@ async function depositReturnedCoins(
       console.error("deposit failed due to status code", resp);
       continue;
     }
-    const respJson = resp.responseJson;
+    const respJson = await resp.json();
     if (respJson.status !== "DEPOSIT_OK") {
       console.error("deposit failed", resp);
       continue;

src/wallet-impl/tip.ts

@@ -41,10 +41,12 @@ export async function getTipStatus(
   tipStatusUrl.searchParams.set("tip_id", res.merchantTipId);
   console.log("checking tip status from", tipStatusUrl.href);
   const merchantResp = await ws.http.get(tipStatusUrl.href);
-  console.log("resp:", merchantResp.responseJson);
+  if (merchantResp.status !== 200) {
-  const tipPickupStatus = TipPickupGetResponse.checked(
+    throw Error(`unexpected status ${merchantResp.status} for tip-pickup`);
-    merchantResp.responseJson,
+  }
-  );
+  const respJson = await merchantResp.json();
+  console.log("resp:", respJson);
+  const tipPickupStatus = TipPickupGetResponse.checked(respJson);
 
   console.log("status", tipPickupStatus);
 
@@ -208,13 +210,16 @@ async function processTipImpl(
   try {
     const req = { planchets: planchetsDetail, tip_id: tipRecord.merchantTipId };
     merchantResp = await ws.http.postJson(tipStatusUrl.href, req);
+    if (merchantResp.status !== 200) {
+      throw Error(`unexpected status ${merchantResp.status} for tip-pickup`);
+    }
     console.log("got merchant resp:", merchantResp);
   } catch (e) {
     console.log("tipping failed", e);
     throw e;
   }
 
-  const response = TipResponse.checked(merchantResp.responseJson);
+  const response = TipResponse.checked(await merchantResp.json());
 
   if (response.reserve_sigs.length !== tipRecord.planchets.length) {
     throw Error("number of tip responses does not match requested planchets");

src/wallet-impl/withdraw.ts

@@ -117,8 +117,12 @@ export async function getWithdrawalInfo(
     throw Error("can't parse URL");
   }
   const resp = await ws.http.get(uriResult.statusUrl);
-  console.log("resp:", resp.responseJson);
+  if (resp.status !== 200) {
-  const status = WithdrawOperationStatusResponse.checked(resp.responseJson);
+    throw Error(`unexpected status (${resp.status}) from bank for ${uriResult.statusUrl}`);
+  }
+  const respJson = await resp.json();
+  console.log("resp:", respJson);
+  const status = WithdrawOperationStatusResponse.checked(respJson);
   return {
     amount: Amounts.parseOrThrow(status.amount),
     confirmTransferUrl: status.confirm_transfer_url,
@@ -228,8 +232,11 @@ async function processPlanchet(
   wd.coin_ev = planchet.coinEv;
   const reqUrl = new URL("reserve/withdraw", exchange.baseUrl).href;
   const resp = await ws.http.postJson(reqUrl, wd);
+  if (resp.status !== 200) {
+    throw Error(`unexpected status ${resp.status} for withdraw`);
+  }
 
-  const r = resp.responseJson;
+  const r = await resp.json();
 
   const denomSig = await ws.cryptoApi.rsaUnblind(
     r.ev_sig,

src/wallet.ts

@@ -22,7 +22,7 @@
 /**
  * Imports.
  */
-import { CryptoApi, CryptoWorkerFactory } from "./crypto/workers/cryptoApi";
+import { CryptoWorkerFactory } from "./crypto/workers/cryptoApi";
 import { HttpRequestLibrary } from "./util/http";
 import {
   oneShotPut,

src/walletTypes.ts

@@ -829,7 +829,7 @@ export class Timestamp {
    * Timestamp in milliseconds.
    */
   @Checkable.Number()
-  t_ms: number;
+  readonly t_ms: number;
 
   static checked: (obj: any) => Timestamp;
 }
@@ -838,7 +838,7 @@ export interface Duration {
   /**
    * Duration in milliseconds.
    */
-  d_ms: number;
+  readonly d_ms: number;
 }
 
 export function getTimestampNow(): Timestamp {

tsconfig.json

@@ -48,6 +48,7 @@
     "src/index.ts",
     "src/talerTypes.ts",
     "src/types-test.ts",
+    "src/util/RequestThrottler.ts",
     "src/util/amounts.ts",
     "src/util/assertUnreachable.ts",
     "src/util/asyncMemo.ts",