From 1d1c847b793620acf3a2b193ab45eabf53234cb2 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Tue, 8 Mar 2022 19:19:29 +0100 Subject: wallet: throttle all http requests even from browsers / service workers --- packages/taler-util/src/RequestThrottler.ts | 149 ++++++++++++++++++++ packages/taler-util/src/index.ts | 1 + .../taler-wallet-core/src/headless/NodeHttpLib.ts | 2 +- .../taler-wallet-core/src/util/RequestThrottler.ts | 156 --------------------- packages/taler-wallet-core/tsconfig.json | 2 +- .../src/browserHttpLib.ts | 44 ++++-- .../src/serviceWorkerHttpLib.ts | 51 ++++--- 7 files changed, 219 insertions(+), 186 deletions(-) create mode 100644 packages/taler-util/src/RequestThrottler.ts delete mode 100644 packages/taler-wallet-core/src/util/RequestThrottler.ts diff --git a/packages/taler-util/src/RequestThrottler.ts b/packages/taler-util/src/RequestThrottler.ts new file mode 100644 index 000000000..7689b4215 --- /dev/null +++ b/packages/taler-util/src/RequestThrottler.ts @@ -0,0 +1,149 @@ +/* + This file is part of GNU Taler + (C) 2019 GNUnet e.V. + + GNU Taler is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNU Taler; see the file COPYING. If not, see + */ + +import { Logger } from "./logging.js"; +import { getTimestampNow, timestampCmp, timestampDifference } from "./time.js"; + +/** + * Implementation of token bucket throttling. + */ + + +const logger = new Logger("RequestThrottler.ts"); + +/** + * Maximum request per second, per origin. + */ +const MAX_PER_SECOND = 100; + +/** + * Maximum request per minute, per origin. + */ +const MAX_PER_MINUTE = 500; + +/** + * Maximum request per hour, per origin. + */ +const MAX_PER_HOUR = 2000; + +/** + * Throttling state for one origin. + */ +class OriginState { + tokensSecond: number = MAX_PER_SECOND; + tokensMinute: number = MAX_PER_MINUTE; + tokensHour: number = MAX_PER_HOUR; + private lastUpdate = getTimestampNow(); + + private refill(): void { + const now = getTimestampNow(); + if (timestampCmp(now, this.lastUpdate) < 0) { + // Did the system time change? + this.lastUpdate = now; + return; + } + const d = timestampDifference(now, this.lastUpdate); + if (d.d_ms === "forever") { + throw Error("assertion failed"); + } + this.tokensSecond = Math.min( + MAX_PER_SECOND, + this.tokensSecond + d.d_ms / 1000, + ); + this.tokensMinute = Math.min( + MAX_PER_MINUTE, + this.tokensMinute + d.d_ms / 1000 / 60, + ); + this.tokensHour = Math.min( + MAX_PER_HOUR, + this.tokensHour + d.d_ms / 1000 / 60 / 60, + ); + this.lastUpdate = now; + } + + /** + * Return true if the request for this origin should be throttled. + * Otherwise, take a token out of the respective buckets. + */ + applyThrottle(): boolean { + this.refill(); + if (this.tokensSecond < 1) { + logger.warn("request throttled (per second limit exceeded)"); + return true; + } + if (this.tokensMinute < 1) { + logger.warn("request throttled (per minute limit exceeded)"); + return true; + } + if (this.tokensHour < 1) { + logger.warn("request throttled (per hour limit exceeded)"); + return true; + } + this.tokensSecond--; + this.tokensMinute--; + this.tokensHour--; + return false; + } +} + +/** + * Request throttler, used as a "last layer of defense" when some + * other part of the re-try logic is broken and we're sending too + * many requests to the same exchange/bank/merchant. + */ +export class RequestThrottler { + private perOriginInfo: { [origin: string]: OriginState } = {}; + + /** + * Get the throttling state for an origin, or + * initialize if no state is associated with the + * origin yet. + */ + private getState(origin: string): OriginState { + const s = this.perOriginInfo[origin]; + if (s) { + return s; + } + const ns = (this.perOriginInfo[origin] = new OriginState()); + return ns; + } + + /** + * Apply throttling to a request. + * + * @returns whether the request should be throttled. + */ + applyThrottle(requestUrl: string): boolean { + const origin = new URL(requestUrl).origin; + return this.getState(origin).applyThrottle(); + } + + /** + * Get the throttle statistics for a particular URL. + */ + getThrottleStats(requestUrl: string): Record { + const origin = new URL(requestUrl).origin; + const state = this.getState(origin); + return { + tokensHour: state.tokensHour, + tokensMinute: state.tokensMinute, + tokensSecond: state.tokensSecond, + maxTokensHour: MAX_PER_HOUR, + maxTokensMinute: MAX_PER_MINUTE, + maxTokensSecond: MAX_PER_SECOND, + }; + } +} diff --git a/packages/taler-util/src/index.ts b/packages/taler-util/src/index.ts index 0141be13b..573b4a5c7 100644 --- a/packages/taler-util/src/index.ts +++ b/packages/taler-util/src/index.ts @@ -30,3 +30,4 @@ export { secretbox_open, crypto_sign_keyPair_fromSeed, } from "./nacl-fast.js"; +export { RequestThrottler } from "./RequestThrottler.js"; diff --git a/packages/taler-wallet-core/src/headless/NodeHttpLib.ts b/packages/taler-wallet-core/src/headless/NodeHttpLib.ts index 5a90994b1..2a8c9e36c 100644 --- a/packages/taler-wallet-core/src/headless/NodeHttpLib.ts +++ b/packages/taler-wallet-core/src/headless/NodeHttpLib.ts @@ -25,7 +25,7 @@ import { HttpRequestOptions, HttpResponse, } from "../util/http.js"; -import { RequestThrottler } from "../util/RequestThrottler.js"; +import { RequestThrottler } from "@gnu-taler/taler-util"; import Axios, { AxiosResponse } from "axios"; import { OperationFailedError, makeErrorDetails } from "../errors.js"; import { Logger, bytesToString } from "@gnu-taler/taler-util"; diff --git a/packages/taler-wallet-core/src/util/RequestThrottler.ts b/packages/taler-wallet-core/src/util/RequestThrottler.ts deleted file mode 100644 index d79afe47a..000000000 --- a/packages/taler-wallet-core/src/util/RequestThrottler.ts +++ /dev/null @@ -1,156 +0,0 @@ -/* - This file is part of GNU Taler - (C) 2019 GNUnet e.V. - - GNU Taler is free software; you can redistribute it and/or modify it under the - terms of the GNU General Public License as published by the Free Software - Foundation; either version 3, or (at your option) any later version. - - TALER is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - A PARTICULAR PURPOSE. See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along with - GNU Taler; see the file COPYING. If not, see - */ - -/** - * Implementation of token bucket throttling. - */ - -/** - * Imports. - */ -import { - getTimestampNow, - timestampDifference, - timestampCmp, - Logger, - URL, -} from "@gnu-taler/taler-util"; - -const logger = new Logger("RequestThrottler.ts"); - -/** - * Maximum request per second, per origin. - */ -const MAX_PER_SECOND = 100; - -/** - * Maximum request per minute, per origin. - */ -const MAX_PER_MINUTE = 500; - -/** - * Maximum request per hour, per origin. - */ -const MAX_PER_HOUR = 2000; - -/** - * Throttling state for one origin. - */ -class OriginState { - tokensSecond: number = MAX_PER_SECOND; - tokensMinute: number = MAX_PER_MINUTE; - tokensHour: number = MAX_PER_HOUR; - private lastUpdate = getTimestampNow(); - - private refill(): void { - const now = getTimestampNow(); - if (timestampCmp(now, this.lastUpdate) < 0) { - // Did the system time change? - this.lastUpdate = now; - return; - } - const d = timestampDifference(now, this.lastUpdate); - if (d.d_ms === "forever") { - throw Error("assertion failed"); - } - this.tokensSecond = Math.min( - MAX_PER_SECOND, - this.tokensSecond + d.d_ms / 1000, - ); - this.tokensMinute = Math.min( - MAX_PER_MINUTE, - this.tokensMinute + d.d_ms / 1000 / 60, - ); - this.tokensHour = Math.min( - MAX_PER_HOUR, - this.tokensHour + d.d_ms / 1000 / 60 / 60, - ); - this.lastUpdate = now; - } - - /** - * Return true if the request for this origin should be throttled. - * Otherwise, take a token out of the respective buckets. - */ - applyThrottle(): boolean { - this.refill(); - if (this.tokensSecond < 1) { - logger.warn("request throttled (per second limit exceeded)"); - return true; - } - if (this.tokensMinute < 1) { - logger.warn("request throttled (per minute limit exceeded)"); - return true; - } - if (this.tokensHour < 1) { - logger.warn("request throttled (per hour limit exceeded)"); - return true; - } - this.tokensSecond--; - this.tokensMinute--; - this.tokensHour--; - return false; - } -} - -/** - * Request throttler, used as a "last layer of defense" when some - * other part of the re-try logic is broken and we're sending too - * many requests to the same exchange/bank/merchant. - */ -export class RequestThrottler { - private perOriginInfo: { [origin: string]: OriginState } = {}; - - /** - * Get the throttling state for an origin, or - * initialize if no state is associated with the - * origin yet. - */ - private getState(origin: string): OriginState { - const s = this.perOriginInfo[origin]; - if (s) { - return s; - } - const ns = (this.perOriginInfo[origin] = new OriginState()); - return ns; - } - - /** - * Apply throttling to a request. - * - * @returns whether the request should be throttled. - */ - applyThrottle(requestUrl: string): boolean { - const origin = new URL(requestUrl).origin; - return this.getState(origin).applyThrottle(); - } - - /** - * Get the throttle statistics for a particular URL. - */ - getThrottleStats(requestUrl: string): Record { - const origin = new URL(requestUrl).origin; - const state = this.getState(origin); - return { - tokensHour: state.tokensHour, - tokensMinute: state.tokensMinute, - tokensSecond: state.tokensSecond, - maxTokensHour: MAX_PER_HOUR, - maxTokensMinute: MAX_PER_MINUTE, - maxTokensSecond: MAX_PER_SECOND, - }; - } -} diff --git a/packages/taler-wallet-core/tsconfig.json b/packages/taler-wallet-core/tsconfig.json index 3da332364..c3366373e 100644 --- a/packages/taler-wallet-core/tsconfig.json +++ b/packages/taler-wallet-core/tsconfig.json @@ -21,7 +21,7 @@ "esModuleInterop": true, "importHelpers": true, "rootDir": "./src", - "typeRoots": ["./node_modules/@types"], + "typeRoots": ["./node_modules/@types"] }, "references": [ { diff --git a/packages/taler-wallet-webextension/src/browserHttpLib.ts b/packages/taler-wallet-webextension/src/browserHttpLib.ts index 63fd456f4..8877edfc3 100644 --- a/packages/taler-wallet-webextension/src/browserHttpLib.ts +++ b/packages/taler-wallet-webextension/src/browserHttpLib.ts @@ -24,7 +24,11 @@ import { HttpResponse, Headers, } from "@gnu-taler/taler-wallet-core"; -import { Logger, TalerErrorCode } from "@gnu-taler/taler-util"; +import { + Logger, + RequestThrottler, + TalerErrorCode, +} from "@gnu-taler/taler-util"; const logger = new Logger("browserHttpLib"); @@ -33,12 +37,32 @@ const logger = new Logger("browserHttpLib"); * browser's XMLHttpRequest. */ export class BrowserHttpLib implements HttpRequestLibrary { - fetch(url: string, options?: HttpRequestOptions): Promise { - const method = options?.method ?? "GET"; + private throttle = new RequestThrottler(); + private throttlingEnabled = true; + + fetch( + requestUrl: string, + options?: HttpRequestOptions, + ): Promise { + const requestMethod = options?.method ?? "GET"; let requestBody = options?.body; + + if (this.throttlingEnabled && this.throttle.applyThrottle(requestUrl)) { + const parsedUrl = new URL(requestUrl); + throw OperationFailedError.fromCode( + TalerErrorCode.WALLET_HTTP_REQUEST_THROTTLED, + `request to origin ${parsedUrl.origin} was throttled`, + { + requestMethod, + requestUrl, + throttleStats: this.throttle.getThrottleStats(requestUrl), + }, + ); + } + return new Promise((resolve, reject) => { const myRequest = new XMLHttpRequest(); - myRequest.open(method, url); + myRequest.open(requestMethod, requestUrl); if (options?.headers) { for (const headerName in options.headers) { myRequest.setRequestHeader(headerName, options.headers[headerName]); @@ -58,7 +82,7 @@ export class BrowserHttpLib implements HttpRequestLibrary { TalerErrorCode.WALLET_NETWORK_ERROR, "Could not make request", { - requestUrl: url, + requestUrl: requestUrl, }, ), ); @@ -71,7 +95,7 @@ export class BrowserHttpLib implements HttpRequestLibrary { TalerErrorCode.WALLET_NETWORK_ERROR, "HTTP request failed (status 0, maybe URI scheme was wrong?)", { - requestUrl: url, + requestUrl: requestUrl, }, ); reject(exc); @@ -92,7 +116,7 @@ export class BrowserHttpLib implements HttpRequestLibrary { TalerErrorCode.WALLET_RECEIVED_MALFORMED_RESPONSE, "Invalid JSON from HTTP response", { - requestUrl: url, + requestUrl: requestUrl, httpStatusCode: myRequest.status, }, ); @@ -102,7 +126,7 @@ export class BrowserHttpLib implements HttpRequestLibrary { TalerErrorCode.WALLET_RECEIVED_MALFORMED_RESPONSE, "Invalid JSON from HTTP response", { - requestUrl: url, + requestUrl: requestUrl, httpStatusCode: myRequest.status, }, ); @@ -126,10 +150,10 @@ export class BrowserHttpLib implements HttpRequestLibrary { headerMap.set(headerName, value); }); const resp: HttpResponse = { - requestUrl: url, + requestUrl: requestUrl, status: myRequest.status, headers: headerMap, - requestMethod: method, + requestMethod: requestMethod, json: makeJson, text: makeText, bytes: async () => myRequest.response, diff --git a/packages/taler-wallet-webextension/src/serviceWorkerHttpLib.ts b/packages/taler-wallet-webextension/src/serviceWorkerHttpLib.ts index a66d4e097..6f2585c1e 100644 --- a/packages/taler-wallet-webextension/src/serviceWorkerHttpLib.ts +++ b/packages/taler-wallet-webextension/src/serviceWorkerHttpLib.ts @@ -17,37 +17,55 @@ /** * Imports. */ -import { Logger, TalerErrorCode } from "@gnu-taler/taler-util"; +import { RequestThrottler, TalerErrorCode } from "@gnu-taler/taler-util"; import { - Headers, HttpRequestLibrary, + Headers, + HttpRequestLibrary, HttpRequestOptions, HttpResponse, - OperationFailedError + OperationFailedError, } from "@gnu-taler/taler-wallet-core"; -const logger = new Logger("browserHttpLib"); - /** * An implementation of the [[HttpRequestLibrary]] using the * browser's XMLHttpRequest. */ export class ServiceWorkerHttpLib implements HttpRequestLibrary { - async fetch(requestUrl: string, options?: HttpRequestOptions): Promise { + private throttle = new RequestThrottler(); + private throttlingEnabled = true; + + async fetch( + requestUrl: string, + options?: HttpRequestOptions, + ): Promise { const requestMethod = options?.method ?? "GET"; const requestBody = options?.body; const requestHeader = options?.headers; + if (this.throttlingEnabled && this.throttle.applyThrottle(requestUrl)) { + const parsedUrl = new URL(requestUrl); + throw OperationFailedError.fromCode( + TalerErrorCode.WALLET_HTTP_REQUEST_THROTTLED, + `request to origin ${parsedUrl.origin} was throttled`, + { + requestMethod, + requestUrl, + throttleStats: this.throttle.getThrottleStats(requestUrl), + }, + ); + } + const response = await fetch(requestUrl, { headers: requestHeader, body: requestBody, method: requestMethod, // timeout: options?.timeout - }) + }); const headerMap = new Headers(); response.headers.forEach((value, key) => { headerMap.set(key, value); - }) + }); return { headers: headerMap, status: response.status, @@ -56,11 +74,9 @@ export class ServiceWorkerHttpLib implements HttpRequestLibrary { json: makeJsonHandler(response, requestUrl), text: makeTextHandler(response, requestUrl), bytes: async () => (await response.blob()).arrayBuffer(), - } - + }; } - get(url: string, opt?: HttpRequestOptions): Promise { return this.fetch(url, { method: "GET", @@ -89,7 +105,7 @@ function makeTextHandler(response: Response, requestUrl: string) { return async function getJsonFromResponse(): Promise { let respText; try { - respText = await response.text() + respText = await response.text(); } catch (e) { throw OperationFailedError.fromCode( TalerErrorCode.WALLET_RECEIVED_MALFORMED_RESPONSE, @@ -100,15 +116,15 @@ function makeTextHandler(response: Response, requestUrl: string) { }, ); } - return respText - } + return respText; + }; } function makeJsonHandler(response: Response, requestUrl: string) { return async function getJsonFromResponse(): Promise { let responseJson; try { - responseJson = await response.json() + responseJson = await response.json(); } catch (e) { throw OperationFailedError.fromCode( TalerErrorCode.WALLET_RECEIVED_MALFORMED_RESPONSE, @@ -129,7 +145,6 @@ function makeJsonHandler(response: Response, requestUrl: string) { }, ); } - return responseJson - } + return responseJson; + }; } - -- cgit v1.2.3