From 421e613f92b80c81c856d6b074aa160e80e38e3d Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Thu, 20 Aug 2020 16:27:20 +0530 Subject: [PATCH] throttling diagnostics and request timeouts --- .../taler-wallet-core/src/TalerErrorCode.ts | 7 ++++ .../src/headless/NodeHttpLib.ts | 16 ++++++++- .../src/operations/exchanges.ts | 19 +++++++--- .../taler-wallet-core/src/operations/pay.ts | 28 +++++++++++++-- .../src/operations/recoup.ts | 6 ++-- .../src/operations/refresh.ts | 28 ++++++++------- .../src/operations/reserves.ts | 36 +++++++++++++++---- .../taler-wallet-core/src/types/dbTypes.ts | 11 ++++++ .../src/util/RequestThrottler.ts | 31 ++++++++++++---- packages/taler-wallet-core/src/util/http.ts | 2 ++ packages/taler-wallet-core/src/util/time.ts | 10 ++++++ 11 files changed, 159 insertions(+), 35 deletions(-) diff --git a/packages/taler-wallet-core/src/TalerErrorCode.ts b/packages/taler-wallet-core/src/TalerErrorCode.ts index 412f3ef8a..ff8511046 100644 --- a/packages/taler-wallet-core/src/TalerErrorCode.ts +++ b/packages/taler-wallet-core/src/TalerErrorCode.ts @@ -3202,6 +3202,13 @@ export enum TalerErrorCode { */ WALLET_WITHDRAWAL_OPERATION_ABORTED_BY_BANK = 7012, + /** + * An HTTP request made by the wallet timed out. + * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0). + * (A value of 0 indicates that the error is generated client-side). + */ + WALLET_HTTP_REQUEST_TIMEOUT = 7013, + /** * End of error code range. * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0). diff --git a/packages/taler-wallet-core/src/headless/NodeHttpLib.ts b/packages/taler-wallet-core/src/headless/NodeHttpLib.ts index 59730ab30..85f37cfa3 100644 --- a/packages/taler-wallet-core/src/headless/NodeHttpLib.ts +++ b/packages/taler-wallet-core/src/headless/NodeHttpLib.ts @@ -29,6 +29,7 @@ import { RequestThrottler } from "../util/RequestThrottler"; import Axios from "axios"; import { OperationFailedError, makeErrorDetails } from "../operations/errors"; import { TalerErrorCode } from "../TalerErrorCode"; +import { URL } from "../util/url"; /** * Implementation of the HTTP request library interface for node. @@ -50,8 +51,20 @@ export class NodeHttpLib implements HttpRequestLibrary { body: any, opt?: HttpRequestOptions, ): Promise { + const parsedUrl = new URL(url); if (this.throttlingEnabled && this.throttle.applyThrottle(url)) { - throw Error("request throttled"); + throw OperationFailedError.fromCode( + TalerErrorCode.WALLET_HTTP_REQUEST_THROTTLED, + `request to origin ${parsedUrl.origin} was throttled`, + { + requestMethod: method, + requestUrl: url, + throttleStats: this.throttle.getThrottleStats(url), + }); + } + let timeout: number | undefined; + if (typeof opt?.timeout?.d_ms === "number") { + timeout = opt.timeout.d_ms; } const resp = await Axios({ method, @@ -61,6 +74,7 @@ export class NodeHttpLib implements HttpRequestLibrary { validateStatus: () => true, transformResponse: (x) => x, data: body, + timeout, }); const respText = resp.data; diff --git a/packages/taler-wallet-core/src/operations/exchanges.ts b/packages/taler-wallet-core/src/operations/exchanges.ts index d40dd7883..3b7f62fe9 100644 --- a/packages/taler-wallet-core/src/operations/exchanges.ts +++ b/packages/taler-wallet-core/src/operations/exchanges.ts @@ -43,7 +43,7 @@ import { WALLET_CACHE_BREAKER_CLIENT_VERSION, WALLET_EXCHANGE_PROTOCOL_VERSION, } from "./versions"; -import { getTimestampNow } from "../util/time"; +import { getTimestampNow, Duration } from "../util/time"; import { compare } from "../util/libtoolVersion"; import { createRecoupGroup, processRecoupGroup } from "./recoup"; import { TalerErrorCode } from "../TalerErrorCode"; @@ -96,6 +96,10 @@ async function setExchangeError( await ws.db.mutate(Stores.exchanges, baseUrl, mut); } +function getExchangeRequestTimeout(e: ExchangeRecord): Duration { + return { d_ms: 5000 }; +} + /** * Fetch the exchange's /keys and update our database accordingly. * @@ -117,7 +121,9 @@ async function updateExchangeWithKeys( const keysUrl = new URL("keys", baseUrl); keysUrl.searchParams.set("cacheBreaker", WALLET_CACHE_BREAKER_CLIENT_VERSION); - const resp = await ws.http.get(keysUrl.href); + const resp = await ws.http.get(keysUrl.href, { + timeout: getExchangeRequestTimeout(existingExchangeRecord), + }); const exchangeKeysJson = await readSuccessResponseJsonOrThrow( resp, codecForExchangeKeysJson(), @@ -303,7 +309,10 @@ async function updateExchangeWithTermsOfService( Accept: "text/plain", }; - const resp = await ws.http.get(reqUrl.href, { headers }); + const resp = await ws.http.get(reqUrl.href, { + headers, + timeout: getExchangeRequestTimeout(exchange), + }); const tosText = await readSuccessResponseTextOrThrow(resp); const tosEtag = resp.headers.get("etag") || undefined; @@ -361,7 +370,9 @@ async function updateExchangeWithWireInfo( const reqUrl = new URL("wire", exchangeBaseUrl); reqUrl.searchParams.set("cacheBreaker", WALLET_CACHE_BREAKER_CLIENT_VERSION); - const resp = await ws.http.get(reqUrl.href); + const resp = await ws.http.get(reqUrl.href, { + timeout: getExchangeRequestTimeout(exchange), + }); const wireInfo = await readSuccessResponseJsonOrThrow( resp, codecForExchangeWireJson(), diff --git a/packages/taler-wallet-core/src/operations/pay.ts b/packages/taler-wallet-core/src/operations/pay.ts index 565fe9c66..f20632344 100644 --- a/packages/taler-wallet-core/src/operations/pay.ts +++ b/packages/taler-wallet-core/src/operations/pay.ts @@ -35,6 +35,7 @@ import { updateRetryInfoTimeout, PayEventRecord, WalletContractData, + getRetryDuration, } from "../types/dbTypes"; import { NotificationType } from "../types/notifications"; import { @@ -58,7 +59,13 @@ import { parsePayUri } from "../util/taleruri"; import { guardOperationException, OperationFailedError } from "./errors"; import { createRefreshGroup, getTotalRefreshCost } from "./refresh"; import { InternalWalletState, EXCHANGE_COINS_LOCK } from "./state"; -import { getTimestampNow, timestampAddDuration } from "../util/time"; +import { + getTimestampNow, + timestampAddDuration, + Duration, + durationMax, + durationMin, +} from "../util/time"; import { strcmp, canonicalJson } from "../util/helpers"; import { readSuccessResponseJsonOrThrow, @@ -588,6 +595,17 @@ async function resetDownloadProposalRetry( }); } +function getProposalRequestTimeout(proposal: ProposalRecord): Duration { + return durationMax( + { d_ms: 60000 }, + durationMin({ d_ms: 5000 }, getRetryDuration(proposal.retryInfo)), + ); +} + +function getPurchaseRequestTimeout(purchase: PurchaseRecord): Duration { + return { d_ms: 5000 }; +} + async function processDownloadProposalImpl( ws: InternalWalletState, proposalId: string, @@ -620,7 +638,9 @@ async function processDownloadProposalImpl( requestBody.token = proposal.claimToken; } - const resp = await ws.http.postJson(orderClaimUrl, requestBody); + const resp = await ws.http.postJson(orderClaimUrl, requestBody, { + timeout: getProposalRequestTimeout(proposal), + }); const proposalResp = await readSuccessResponseJsonOrThrow( resp, codecForProposal(), @@ -886,7 +906,9 @@ export async function submitPay( logger.trace("making pay request", JSON.stringify(reqBody, undefined, 2)); const resp = await ws.runSequentialized([EXCHANGE_COINS_LOCK], () => - ws.http.postJson(payUrl, reqBody), + ws.http.postJson(payUrl, reqBody, { + timeout: getPurchaseRequestTimeout(purchase), + }), ); const merchantResp = await readSuccessResponseJsonOrThrow( diff --git a/packages/taler-wallet-core/src/operations/recoup.ts b/packages/taler-wallet-core/src/operations/recoup.ts index f855a28cb..ba02f72f8 100644 --- a/packages/taler-wallet-core/src/operations/recoup.ts +++ b/packages/taler-wallet-core/src/operations/recoup.ts @@ -40,7 +40,7 @@ import { import { codecForRecoupConfirmation } from "../types/talerTypes"; import { NotificationType } from "../types/notifications"; -import { forceQueryReserve } from "./reserves"; +import { forceQueryReserve, getReserveRequestTimeout } from "./reserves"; import { Amounts } from "../util/amounts"; import { createRefreshGroup, processRefreshGroup } from "./refresh"; @@ -154,7 +154,9 @@ async function recoupWithdrawCoin( const recoupRequest = await ws.cryptoApi.createRecoupRequest(coin); const reqUrl = new URL(`/coins/${coin.coinPub}/recoup`, coin.exchangeBaseUrl); - const resp = await ws.http.postJson(reqUrl.href, recoupRequest); + const resp = await ws.http.postJson(reqUrl.href, recoupRequest, { + timeout: getReserveRequestTimeout(reserve), + }); const recoupConfirmation = await readSuccessResponseJsonOrThrow( resp, codecForRecoupConfirmation(), diff --git a/packages/taler-wallet-core/src/operations/refresh.ts b/packages/taler-wallet-core/src/operations/refresh.ts index 430675328..89cc3af43 100644 --- a/packages/taler-wallet-core/src/operations/refresh.ts +++ b/packages/taler-wallet-core/src/operations/refresh.ts @@ -42,7 +42,7 @@ import { import { guardOperationException } from "./errors"; import { NotificationType } from "../types/notifications"; import { getRandomBytes, encodeCrock } from "../crypto/talerCrypto"; -import { getTimestampNow } from "../util/time"; +import { getTimestampNow, Duration } from "../util/time"; import { readSuccessResponseJsonOrThrow, HttpResponse } from "../util/http"; import { codecForExchangeMeltResponse, @@ -211,6 +211,10 @@ async function refreshCreateSession( ws.notify({ type: NotificationType.RefreshStarted }); } +function getRefreshRequestTimeout(rg: RefreshGroupRecord): Duration { + return { d_ms: 5000 }; +} + async function refreshMelt( ws: InternalWalletState, refreshGroupId: string, @@ -249,12 +253,11 @@ async function refreshMelt( }; logger.trace(`melt request for coin:`, meltReq); - const resp = await ws.runSequentialized( - [EXCHANGE_COINS_LOCK], - async () => { - return await ws.http.postJson(reqUrl.href, meltReq); - }, - ); + const resp = await ws.runSequentialized([EXCHANGE_COINS_LOCK], async () => { + return await ws.http.postJson(reqUrl.href, meltReq, { + timeout: getRefreshRequestTimeout(refreshGroup), + }); + }); const meltResponse = await readSuccessResponseJsonOrThrow( resp, @@ -346,12 +349,11 @@ async function refreshReveal( refreshSession.exchangeBaseUrl, ); - const resp = await ws.runSequentialized( - [EXCHANGE_COINS_LOCK], - async () => { - return await ws.http.postJson(reqUrl.href, req); - }, - ); + const resp = await ws.runSequentialized([EXCHANGE_COINS_LOCK], async () => { + return await ws.http.postJson(reqUrl.href, req, { + timeout: getRefreshRequestTimeout(refreshGroup), + }); + }); const reveal = await readSuccessResponseJsonOrThrow( resp, diff --git a/packages/taler-wallet-core/src/operations/reserves.ts b/packages/taler-wallet-core/src/operations/reserves.ts index 8adaeea81..fda3c4bcb 100644 --- a/packages/taler-wallet-core/src/operations/reserves.ts +++ b/packages/taler-wallet-core/src/operations/reserves.ts @@ -35,6 +35,7 @@ import { WithdrawalSourceType, ReserveHistoryRecord, ReserveBankInfo, + getRetryDuration, } from "../types/dbTypes"; import { Logger } from "../util/logging"; import { Amounts } from "../util/amounts"; @@ -64,7 +65,12 @@ import { } from "./errors"; import { NotificationType } from "../types/notifications"; import { codecForReserveStatus } from "../types/ReserveStatus"; -import { getTimestampNow } from "../util/time"; +import { + getTimestampNow, + Duration, + durationMin, + durationMax, +} from "../util/time"; import { reconcileReserveHistory, summarizeReserveHistory, @@ -331,10 +337,16 @@ async function registerReserveWithBank( return; } const bankStatusUrl = bankInfo.statusUrl; - const httpResp = await ws.http.postJson(bankStatusUrl, { - reserve_pub: reservePub, - selected_exchange: bankInfo.exchangePaytoUri, - }); + const httpResp = await ws.http.postJson( + bankStatusUrl, + { + reserve_pub: reservePub, + selected_exchange: bankInfo.exchangePaytoUri, + }, + { + timeout: getReserveRequestTimeout(reserve), + }, + ); await readSuccessResponseJsonOrThrow( httpResp, codecForBankWithdrawalOperationPostResponse(), @@ -371,6 +383,13 @@ async function processReserveBankStatus( ); } +export function getReserveRequestTimeout(r: ReserveRecord): Duration { + return durationMax( + { d_ms: 60000 }, + durationMin({ d_ms: 5000 }, getRetryDuration(r.retryInfo)), + ); +} + async function processReserveBankStatusImpl( ws: InternalWalletState, reservePub: string, @@ -388,7 +407,9 @@ async function processReserveBankStatusImpl( return; } - const statusResp = await ws.http.get(bankStatusUrl); + const statusResp = await ws.http.get(bankStatusUrl, { + timeout: getReserveRequestTimeout(reserve), + }); const status = await readSuccessResponseJsonOrThrow( statusResp, codecForWithdrawOperationStatusResponse(), @@ -501,6 +522,9 @@ async function updateReserve( const resp = await ws.http.get( new URL(`reserves/${reservePub}`, reserve.exchangeBaseUrl).href, + { + timeout: getReserveRequestTimeout(reserve), + }, ); const result = await readSuccessResponseJsonOrErrorCode( diff --git a/packages/taler-wallet-core/src/types/dbTypes.ts b/packages/taler-wallet-core/src/types/dbTypes.ts index 82260963b..e36e322d1 100644 --- a/packages/taler-wallet-core/src/types/dbTypes.ts +++ b/packages/taler-wallet-core/src/types/dbTypes.ts @@ -117,6 +117,17 @@ export function updateRetryInfoTimeout( r.nextRetry = { t_ms: t }; } +export function getRetryDuration( + r: RetryInfo, + p: RetryPolicy = defaultRetryPolicy, +): Duration { + if (p.backoffDelta.d_ms === "forever") { + return { d_ms: "forever" }; + } + const t = p.backoffDelta.d_ms * Math.pow(p.backoffBase, r.retryCounter); + return { d_ms: t }; +} + export function initRetryInfo( active = true, p: RetryPolicy = defaultRetryPolicy, diff --git a/packages/taler-wallet-core/src/util/RequestThrottler.ts b/packages/taler-wallet-core/src/util/RequestThrottler.ts index 3b8f22f58..b56f7476a 100644 --- a/packages/taler-wallet-core/src/util/RequestThrottler.ts +++ b/packages/taler-wallet-core/src/util/RequestThrottler.ts @@ -30,25 +30,25 @@ const logger = new Logger("RequestThrottler.ts"); /** * Maximum request per second, per origin. */ -const MAX_PER_SECOND = 50; +const MAX_PER_SECOND = 100; /** * Maximum request per minute, per origin. */ -const MAX_PER_MINUTE = 100; +const MAX_PER_MINUTE = 500; /** * Maximum request per hour, per origin. */ -const MAX_PER_HOUR = 1000; +const MAX_PER_HOUR = 2000; /** * Throttling state for one origin. */ class OriginState { - private tokensSecond: number = MAX_PER_SECOND; - private tokensMinute: number = MAX_PER_MINUTE; - private tokensHour: number = MAX_PER_HOUR; + tokensSecond: number = MAX_PER_SECOND; + tokensMinute: number = MAX_PER_MINUTE; + tokensHour: number = MAX_PER_HOUR; private lastUpdate = getTimestampNow(); private refill(): void { @@ -57,6 +57,9 @@ class OriginState { if (d.d_ms === "forever") { throw Error("assertion failed"); } + if (d.d_ms < 0) { + return; + } const d_s = d.d_ms / 1000; this.tokensSecond = Math.min( MAX_PER_SECOND, @@ -129,4 +132,20 @@ export class RequestThrottler { const origin = new URL(requestUrl).origin; return this.getState(origin).applyThrottle(); } + + /** + * Get the throttle statistics for a particular URL. + */ + getThrottleStats(requestUrl: string): Record { + const origin = new URL(requestUrl).origin; + const state = this.getState(origin); + return { + tokensHour: state.tokensHour, + tokensMinute: state.tokensMinute, + tokensSecond: state.tokensSecond, + maxTokensHour: MAX_PER_HOUR, + maxTokensMinute: MAX_PER_MINUTE, + maxTokensSecond: MAX_PER_SECOND, + } + } } diff --git a/packages/taler-wallet-core/src/util/http.ts b/packages/taler-wallet-core/src/util/http.ts index 22566daac..44c01a4e5 100644 --- a/packages/taler-wallet-core/src/util/http.ts +++ b/packages/taler-wallet-core/src/util/http.ts @@ -26,6 +26,7 @@ import { Codec } from "./codec"; import { OperationFailedError, makeErrorDetails } from "../operations/errors"; import { TalerErrorCode } from "../TalerErrorCode"; import { Logger } from "./logging"; +import { Duration } from "./time"; const logger = new Logger("http.ts"); @@ -43,6 +44,7 @@ export interface HttpResponse { export interface HttpRequestOptions { headers?: { [name: string]: string }; + timeout?: Duration, } export enum HttpResponseStatus { diff --git a/packages/taler-wallet-core/src/util/time.ts b/packages/taler-wallet-core/src/util/time.ts index 5c2f49d12..ccd75e14b 100644 --- a/packages/taler-wallet-core/src/util/time.ts +++ b/packages/taler-wallet-core/src/util/time.ts @@ -95,6 +95,16 @@ export function durationMin(d1: Duration, d2: Duration): Duration { return { d_ms: Math.min(d1.d_ms, d2.d_ms) }; } +export function durationMax(d1: Duration, d2: Duration): Duration { + if (d1.d_ms === "forever") { + return { d_ms: "forever" }; + } + if (d2.d_ms === "forever") { + return { d_ms: "forever" }; + } + return { d_ms: Math.max(d1.d_ms, d2.d_ms) }; +} + export function timestampCmp(t1: Timestamp, t2: Timestamp): number { if (t1.t_ms === "never") { if (t2.t_ms === "never") {