From 175f5b24957fca79484da05750e10b5e0905aa1a Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Tue, 7 Dec 2021 13:57:37 +0100 Subject: [PATCH 01/37] -comment --- src/exchange/taler-exchange-httpd_db.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/exchange/taler-exchange-httpd_db.c b/src/exchange/taler-exchange-httpd_db.c index fb876f92b..08b578445 100644 --- a/src/exchange/taler-exchange-httpd_db.c +++ b/src/exchange/taler-exchange-httpd_db.c @@ -98,6 +98,8 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin, // FIXME: why do we even return the transaction // history here!? This is a coin with multiple // associated denominations, after all... + // => this is probably the wrong call, as this + // is NOT about insufficient funds! *mhd_ret = TEH_RESPONSE_reply_coin_insufficient_funds ( connection, From 8be9de667525a307c35e12ecacc0c3a647ccdd9b Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Tue, 7 Dec 2021 21:02:36 +0100 Subject: [PATCH 02/37] implement dummy /metrics handler --- src/exchange/Makefile.am | 1 + src/exchange/taler-exchange-httpd.c | 7 ++++ src/exchange/taler-exchange-httpd_metrics.c | 43 +++++++++++++++++++++ src/exchange/taler-exchange-httpd_metrics.h | 41 ++++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 src/exchange/taler-exchange-httpd_metrics.c create mode 100644 src/exchange/taler-exchange-httpd_metrics.h diff --git a/src/exchange/Makefile.am b/src/exchange/Makefile.am index 7779c38b1..3009c84c2 100644 --- a/src/exchange/Makefile.am +++ b/src/exchange/Makefile.am @@ -97,6 +97,7 @@ taler_exchange_httpd_SOURCES = \ taler-exchange-httpd_management_wire_disable.c \ taler-exchange-httpd_management_wire_fees.c \ taler-exchange-httpd_melt.c taler-exchange-httpd_melt.h \ + taler-exchange-httpd_metrics.c taler-exchange-httpd_metrics.h \ taler-exchange-httpd_mhd.c taler-exchange-httpd_mhd.h \ taler-exchange-httpd_recoup.c taler-exchange-httpd_recoup.h \ taler-exchange-httpd_refreshes_reveal.c taler-exchange-httpd_refreshes_reveal.h \ diff --git a/src/exchange/taler-exchange-httpd.c b/src/exchange/taler-exchange-httpd.c index 6bda5821b..57c965189 100644 --- a/src/exchange/taler-exchange-httpd.c +++ b/src/exchange/taler-exchange-httpd.c @@ -39,6 +39,7 @@ #include "taler-exchange-httpd_link.h" #include "taler-exchange-httpd_management.h" #include "taler-exchange-httpd_melt.h" +#include "taler-exchange-httpd_metrics.h" #include "taler-exchange-httpd_mhd.h" #include "taler-exchange-httpd_recoup.h" #include "taler-exchange-httpd_refreshes_reveal.h" @@ -849,6 +850,12 @@ handle_mhd_request (void *cls, .method = MHD_HTTP_METHOD_GET, .handler.get = &handler_seed }, + /* Performance metrics */ + { + .url = "metrics", + .method = MHD_HTTP_METHOD_GET, + .handler.get = &TEH_handler_metrics + }, /* Terms of service */ { .url = "terms", diff --git a/src/exchange/taler-exchange-httpd_metrics.c b/src/exchange/taler-exchange-httpd_metrics.c new file mode 100644 index 000000000..953210926 --- /dev/null +++ b/src/exchange/taler-exchange-httpd_metrics.c @@ -0,0 +1,43 @@ +/* + This file is part of TALER + Copyright (C) 2015-2021 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU Affero General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file taler-exchange-httpd_metrics.c + * @brief Handle /metrics requests + * @author Christian Grothoff + */ +#include "platform.h" +#include +#include "taler_dbevents.h" +#include "taler-exchange-httpd_responses.h" +#include "taler-exchange-httpd_keys.h" +#include "taler-exchange-httpd_metrics.h" +#include "taler_json_lib.h" +#include "taler_mhd_lib.h" +#include + + +MHD_RESULT +TEH_handler_metrics (struct TEH_RequestContext *rc, + const char *const args[]) +{ + (void) args; + return TALER_MHD_reply_json (rc->connection, + json_pack ("{}"), + MHD_HTTP_NO_CONTENT); +} + + +/* end of taler-exchange-httpd_metrics.c */ diff --git a/src/exchange/taler-exchange-httpd_metrics.h b/src/exchange/taler-exchange-httpd_metrics.h new file mode 100644 index 000000000..399a17670 --- /dev/null +++ b/src/exchange/taler-exchange-httpd_metrics.h @@ -0,0 +1,41 @@ +/* + This file is part of TALER + Copyright (C) 2014--2021 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU Affero General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file taler-exchange-httpd_metrics.h + * @brief Handle /metrics requests + * @author Christian Grothoff + */ +#ifndef TALER_EXCHANGE_HTTPD_METRICS_H +#define TALER_EXCHANGE_HTTPD_METRICS_H + +#include +#include +#include "taler-exchange-httpd.h" + + +/** + * Handle a "/metrics" request. + * + * @param rc request context + * @param args array of additional options (must be empty for this function) + * @return MHD result code + */ +MHD_RESULT +TEH_handler_metrics (struct TEH_RequestContext *rc, + const char *const args[]); + + +#endif From 71933dd2e9b5b66dbc72cdde9ea7b498a11db197 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Tue, 7 Dec 2021 21:16:38 +0100 Subject: [PATCH 03/37] add basic metrics collection logic --- src/exchange/taler-exchange-httpd_auditors.c | 1 + src/exchange/taler-exchange-httpd_db.c | 21 ++++---------- src/exchange/taler-exchange-httpd_db.h | 3 ++ src/exchange/taler-exchange-httpd_deposit.c | 1 + .../taler-exchange-httpd_deposits_get.c | 1 + src/exchange/taler-exchange-httpd_kyc-check.c | 1 + src/exchange/taler-exchange-httpd_kyc-proof.c | 1 + .../taler-exchange-httpd_kyc-wallet.c | 1 + src/exchange/taler-exchange-httpd_link.c | 1 + ...taler-exchange-httpd_management_auditors.c | 1 + ...nge-httpd_management_auditors_AP_disable.c | 1 + ...aler-exchange-httpd_management_post_keys.c | 1 + ...r-exchange-httpd_management_wire_disable.c | 1 + ...er-exchange-httpd_management_wire_enable.c | 1 + ...aler-exchange-httpd_management_wire_fees.c | 1 + src/exchange/taler-exchange-httpd_melt.c | 1 + src/exchange/taler-exchange-httpd_metrics.c | 5 ++++ src/exchange/taler-exchange-httpd_metrics.h | 28 +++++++++++++++++++ src/exchange/taler-exchange-httpd_recoup.c | 1 + .../taler-exchange-httpd_refreshes_reveal.c | 3 ++ src/exchange/taler-exchange-httpd_refund.c | 1 + .../taler-exchange-httpd_reserves_get.c | 1 + .../taler-exchange-httpd_transfers_get.c | 1 + src/exchange/taler-exchange-httpd_withdraw.c | 1 + 24 files changed, 63 insertions(+), 16 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_auditors.c b/src/exchange/taler-exchange-httpd_auditors.c index bf4a9b2c0..1b8af311c 100644 --- a/src/exchange/taler-exchange-httpd_auditors.c +++ b/src/exchange/taler-exchange-httpd_auditors.c @@ -216,6 +216,7 @@ TEH_handler_auditors ( return MHD_YES; /* failure */ ret = TEH_DB_run_transaction (connection, "add auditor denom sig", + TEH_MT_OTHER, &res, &add_auditor_denom_sig, &awc); diff --git a/src/exchange/taler-exchange-httpd_db.c b/src/exchange/taler-exchange-httpd_db.c index 08b578445..da495517a 100644 --- a/src/exchange/taler-exchange-httpd_db.c +++ b/src/exchange/taler-exchange-httpd_db.c @@ -24,6 +24,7 @@ #include #include "taler_json_lib.h" #include "taler_mhd_lib.h" +#include "taler-exchange-httpd_db.h" #include "taler-exchange-httpd_responses.h" @@ -113,25 +114,10 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin, } -/** - * Run a database transaction for @a connection. - * Starts a transaction and calls @a cb. Upon success, - * attempts to commit the transaction. Upon soft failures, - * retries @a cb a few times. Upon hard or persistent soft - * errors, generates an error message for @a connection. - * - * @param connection MHD connection to run @a cb for, can be NULL - * @param name name of the transaction (for debugging) - * @param[out] mhd_ret set to MHD response code, if transaction failed; - * NULL if we are not running with a @a connection and thus - * must not queue MHD replies - * @param cb callback implementing transaction logic - * @param cb_cls closure for @a cb, must be read-only! - * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure - */ enum GNUNET_GenericReturnValue TEH_DB_run_transaction (struct MHD_Connection *connection, const char *name, + enum TEH_MetricType mt, MHD_RESULT *mhd_ret, TEH_DB_TransactionCallback cb, void *cb_cls) @@ -149,6 +135,8 @@ TEH_DB_run_transaction (struct MHD_Connection *connection, NULL); return GNUNET_SYSERR; } + GNUNET_assert (mt < TEH_MT_COUNT); + TEH_METRICS_num_requests[mt]++; for (unsigned int retries = 0; retries < MAX_TRANSACTION_COMMIT_RETRIES; retries++) @@ -190,6 +178,7 @@ TEH_DB_run_transaction (struct MHD_Connection *connection, (-1 == (int) *mhd_ret) ); if (0 <= qs) return GNUNET_OK; + TEH_METRICS_num_conflict[mt]++; } TALER_LOG_ERROR ("Transaction `%s' commit failed %u times\n", name, diff --git a/src/exchange/taler-exchange-httpd_db.h b/src/exchange/taler-exchange-httpd_db.h index c115981d3..83302235d 100644 --- a/src/exchange/taler-exchange-httpd_db.h +++ b/src/exchange/taler-exchange-httpd_db.h @@ -23,6 +23,7 @@ #include #include "taler_exchangedb_plugin.h" +#include "taler-exchange-httpd_metrics.h" #include @@ -69,6 +70,7 @@ typedef enum GNUNET_DB_QueryStatus * * @param connection MHD connection to run @a cb for, can be NULL * @param name name of the transaction (for debugging) + * @param mt type of the requests, for metric generation * @param[out] mhd_ret set to MHD response code, if transaction failed (returned #GNUNET_SYSERR); * NULL if we are not running with a @a connection and thus * must not queue MHD replies @@ -79,6 +81,7 @@ typedef enum GNUNET_DB_QueryStatus enum GNUNET_GenericReturnValue TEH_DB_run_transaction (struct MHD_Connection *connection, const char *name, + enum TEH_MetricType mt, MHD_RESULT *mhd_ret, TEH_DB_TransactionCallback cb, void *cb_cls); diff --git a/src/exchange/taler-exchange-httpd_deposit.c b/src/exchange/taler-exchange-httpd_deposit.c index 6b651f40e..5534c402e 100644 --- a/src/exchange/taler-exchange-httpd_deposit.c +++ b/src/exchange/taler-exchange-httpd_deposit.c @@ -497,6 +497,7 @@ TEH_handler_deposit (struct MHD_Connection *connection, if (GNUNET_OK != TEH_DB_run_transaction (connection, "execute deposit", + TEH_MT_DEPOSIT, &mhd_ret, &deposit_transaction, &dc)) diff --git a/src/exchange/taler-exchange-httpd_deposits_get.c b/src/exchange/taler-exchange-httpd_deposits_get.c index d981a8dd9..3db177015 100644 --- a/src/exchange/taler-exchange-httpd_deposits_get.c +++ b/src/exchange/taler-exchange-httpd_deposits_get.c @@ -246,6 +246,7 @@ handle_track_transaction_request ( if (GNUNET_OK != TEH_DB_run_transaction (connection, "handle deposits GET", + TEH_MT_OTHER, &mhd_ret, &deposits_get_transaction, &ctx)) diff --git a/src/exchange/taler-exchange-httpd_kyc-check.c b/src/exchange/taler-exchange-httpd_kyc-check.c index 76d094817..1edbbf2aa 100644 --- a/src/exchange/taler-exchange-httpd_kyc-check.c +++ b/src/exchange/taler-exchange-httpd_kyc-check.c @@ -351,6 +351,7 @@ TEH_handler_kyc_check ( (void) GNUNET_TIME_round_abs (&now); ret = TEH_DB_run_transaction (rc->connection, "kyc check", + TEH_MT_OTHER, &res, &kyc_check, kyp); diff --git a/src/exchange/taler-exchange-httpd_kyc-proof.c b/src/exchange/taler-exchange-httpd_kyc-proof.c index 6bd98abfe..24ddfc74d 100644 --- a/src/exchange/taler-exchange-httpd_kyc-proof.c +++ b/src/exchange/taler-exchange-httpd_kyc-proof.c @@ -678,6 +678,7 @@ TEH_handler_kyc_proof ( ret = TEH_DB_run_transaction (kpc->rc->connection, "check proof kyc", + TEH_MT_OTHER, &res, &persist_kyc_ok, kpc); diff --git a/src/exchange/taler-exchange-httpd_kyc-wallet.c b/src/exchange/taler-exchange-httpd_kyc-wallet.c index 3db174bf0..4062f9305 100644 --- a/src/exchange/taler-exchange-httpd_kyc-wallet.c +++ b/src/exchange/taler-exchange-httpd_kyc-wallet.c @@ -140,6 +140,7 @@ TEH_handler_kyc_wallet ( 0); ret = TEH_DB_run_transaction (rc->connection, "check wallet kyc", + TEH_MT_OTHER, &res, &wallet_kyc_check, &krc); diff --git a/src/exchange/taler-exchange-httpd_link.c b/src/exchange/taler-exchange-httpd_link.c index 3393e0683..d3c0d6a5a 100644 --- a/src/exchange/taler-exchange-httpd_link.c +++ b/src/exchange/taler-exchange-httpd_link.c @@ -193,6 +193,7 @@ TEH_handler_link (struct TEH_RequestContext *rc, if (GNUNET_OK != TEH_DB_run_transaction (rc->connection, "run link", + TEH_MT_OTHER, &mhd_ret, &link_transaction, &ctx)) diff --git a/src/exchange/taler-exchange-httpd_management_auditors.c b/src/exchange/taler-exchange-httpd_management_auditors.c index d782618f1..f9092c53b 100644 --- a/src/exchange/taler-exchange-httpd_management_auditors.c +++ b/src/exchange/taler-exchange-httpd_management_auditors.c @@ -187,6 +187,7 @@ TEH_handler_management_auditors ( ret = TEH_DB_run_transaction (connection, "add auditor", + TEH_MT_OTHER, &res, &add_auditor, &aac); diff --git a/src/exchange/taler-exchange-httpd_management_auditors_AP_disable.c b/src/exchange/taler-exchange-httpd_management_auditors_AP_disable.c index 7bf191f47..8b31fb139 100644 --- a/src/exchange/taler-exchange-httpd_management_auditors_AP_disable.c +++ b/src/exchange/taler-exchange-httpd_management_auditors_AP_disable.c @@ -176,6 +176,7 @@ TEH_handler_management_auditors_AP_disable ( ret = TEH_DB_run_transaction (connection, "del auditor", + TEH_MT_OTHER, &res, &del_auditor, &dac); diff --git a/src/exchange/taler-exchange-httpd_management_post_keys.c b/src/exchange/taler-exchange-httpd_management_post_keys.c index 86b17cb3d..ad4cd3c04 100644 --- a/src/exchange/taler-exchange-httpd_management_post_keys.c +++ b/src/exchange/taler-exchange-httpd_management_post_keys.c @@ -454,6 +454,7 @@ TEH_handler_management_post_keys ( res = TEH_DB_run_transaction (connection, "add keys", + TEH_MT_OTHER, &ret, &add_keys, &akc); diff --git a/src/exchange/taler-exchange-httpd_management_wire_disable.c b/src/exchange/taler-exchange-httpd_management_wire_disable.c index 17bd7273f..2d37a7204 100644 --- a/src/exchange/taler-exchange-httpd_management_wire_disable.c +++ b/src/exchange/taler-exchange-httpd_management_wire_disable.c @@ -177,6 +177,7 @@ TEH_handler_management_post_wire_disable ( res = TEH_DB_run_transaction (connection, "del wire", + TEH_MT_OTHER, &ret, &del_wire, &awc); diff --git a/src/exchange/taler-exchange-httpd_management_wire_enable.c b/src/exchange/taler-exchange-httpd_management_wire_enable.c index 4bad41b40..165c5183d 100644 --- a/src/exchange/taler-exchange-httpd_management_wire_enable.c +++ b/src/exchange/taler-exchange-httpd_management_wire_enable.c @@ -212,6 +212,7 @@ TEH_handler_management_post_wire ( res = TEH_DB_run_transaction (connection, "add wire", + TEH_MT_OTHER, &ret, &add_wire, &awc); diff --git a/src/exchange/taler-exchange-httpd_management_wire_fees.c b/src/exchange/taler-exchange-httpd_management_wire_fees.c index 4272a2d06..180149042 100644 --- a/src/exchange/taler-exchange-httpd_management_wire_fees.c +++ b/src/exchange/taler-exchange-httpd_management_wire_fees.c @@ -221,6 +221,7 @@ TEH_handler_management_post_wire_fees ( res = TEH_DB_run_transaction (connection, "add wire fee", + TEH_MT_OTHER, &ret, &add_fee, &afc); diff --git a/src/exchange/taler-exchange-httpd_melt.c b/src/exchange/taler-exchange-httpd_melt.c index 3dbff43a8..e2e343711 100644 --- a/src/exchange/taler-exchange-httpd_melt.c +++ b/src/exchange/taler-exchange-httpd_melt.c @@ -432,6 +432,7 @@ handle_melt (struct MHD_Connection *connection, if (GNUNET_OK != TEH_DB_run_transaction (connection, "run melt", + TEH_MT_MELT, &mhd_ret, &melt_transaction, rmc)) diff --git a/src/exchange/taler-exchange-httpd_metrics.c b/src/exchange/taler-exchange-httpd_metrics.c index 953210926..d3f1f02d8 100644 --- a/src/exchange/taler-exchange-httpd_metrics.c +++ b/src/exchange/taler-exchange-httpd_metrics.c @@ -29,6 +29,11 @@ #include +unsigned long long TEH_METRICS_num_requests[TEH_MT_COUNT]; + +unsigned long long TEH_METRICS_num_conflict[TEH_MT_COUNT]; + + MHD_RESULT TEH_handler_metrics (struct TEH_RequestContext *rc, const char *const args[]) diff --git a/src/exchange/taler-exchange-httpd_metrics.h b/src/exchange/taler-exchange-httpd_metrics.h index 399a17670..39e463169 100644 --- a/src/exchange/taler-exchange-httpd_metrics.h +++ b/src/exchange/taler-exchange-httpd_metrics.h @@ -26,6 +26,34 @@ #include "taler-exchange-httpd.h" +/** + * Request types for which we collect metrics. + */ +enum TEH_MetricType +{ + TEH_MT_OTHER = 0, + TEH_MT_DEPOSIT = 1, + TEH_MT_WITHDRAW = 2, + TEH_MT_MELT = 3, + TEH_MT_REVEAL_PRECHECK = 4, + TEH_MT_REVEAL = 5, + TEH_MT_REVEAL_PERSIST = 6, + TEH_MT_COUNT = 7 /* MUST BE LAST! */ +}; + + +/** + * Number of requests handled of the respective type. + */ +extern unsigned long long TEH_METRICS_num_requests[TEH_MT_COUNT]; + +/** + * Number of serialization errors encountered when + * handling requests of the respective type. + */ +extern unsigned long long TEH_METRICS_num_conflict[TEH_MT_COUNT]; + + /** * Handle a "/metrics" request. * diff --git a/src/exchange/taler-exchange-httpd_recoup.c b/src/exchange/taler-exchange-httpd_recoup.c index b5074ce35..7ea345b87 100644 --- a/src/exchange/taler-exchange-httpd_recoup.c +++ b/src/exchange/taler-exchange-httpd_recoup.c @@ -477,6 +477,7 @@ verify_and_execute_recoup ( if (GNUNET_OK != TEH_DB_run_transaction (connection, "run recoup", + TEH_MT_OTHER, &mhd_ret, &recoup_transaction, &pc)) diff --git a/src/exchange/taler-exchange-httpd_refreshes_reveal.c b/src/exchange/taler-exchange-httpd_refreshes_reveal.c index 4631a2b92..08587a516 100644 --- a/src/exchange/taler-exchange-httpd_refreshes_reveal.c +++ b/src/exchange/taler-exchange-httpd_refreshes_reveal.c @@ -724,6 +724,7 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection, if ( (GNUNET_OK == TEH_DB_run_transaction (connection, "reveal pre-check", + TEH_MT_REVEAL_PRECHECK, &ret, &refreshes_reveal_preflight, rctx)) && @@ -745,6 +746,7 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection, if (GNUNET_OK != TEH_DB_run_transaction (connection, "run reveal", + TEH_MT_REVEAL, &ret, &refreshes_reveal_transaction, rctx)) @@ -756,6 +758,7 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection, if (GNUNET_OK == TEH_DB_run_transaction (connection, "persist reveal", + TEH_MT_REVEAL_PERSIST, &ret, &refreshes_reveal_persist, rctx)) diff --git a/src/exchange/taler-exchange-httpd_refund.c b/src/exchange/taler-exchange-httpd_refund.c index be8a88df2..a1ef50a32 100644 --- a/src/exchange/taler-exchange-httpd_refund.c +++ b/src/exchange/taler-exchange-httpd_refund.c @@ -447,6 +447,7 @@ verify_and_execute_refund (struct MHD_Connection *connection, if (GNUNET_OK != TEH_DB_run_transaction (connection, "run refund", + TEH_MT_OTHER, &mhd_ret, &refund_transaction, (void *) refund)) diff --git a/src/exchange/taler-exchange-httpd_reserves_get.c b/src/exchange/taler-exchange-httpd_reserves_get.c index 3b8354215..80c992e61 100644 --- a/src/exchange/taler-exchange-httpd_reserves_get.c +++ b/src/exchange/taler-exchange-httpd_reserves_get.c @@ -315,6 +315,7 @@ TEH_handler_reserves_get (struct TEH_RequestContext *rc, if (GNUNET_OK != TEH_DB_run_transaction (rc->connection, "get reserve history", + TEH_MT_OTHER, &mhd_ret, &reserve_history_transaction, &rsc)) diff --git a/src/exchange/taler-exchange-httpd_transfers_get.c b/src/exchange/taler-exchange-httpd_transfers_get.c index e63acdc2a..38a5c211d 100644 --- a/src/exchange/taler-exchange-httpd_transfers_get.c +++ b/src/exchange/taler-exchange-httpd_transfers_get.c @@ -515,6 +515,7 @@ TEH_handler_transfers_get (struct TEH_RequestContext *rc, if (GNUNET_OK != TEH_DB_run_transaction (rc->connection, "run transfers GET", + TEH_MT_OTHER, &mhd_ret, &get_transfer_deposits, &ctx)) diff --git a/src/exchange/taler-exchange-httpd_withdraw.c b/src/exchange/taler-exchange-httpd_withdraw.c index 8540fca4b..9a45271bb 100644 --- a/src/exchange/taler-exchange-httpd_withdraw.c +++ b/src/exchange/taler-exchange-httpd_withdraw.c @@ -521,6 +521,7 @@ TEH_handler_withdraw (struct TEH_RequestContext *rc, if (GNUNET_OK != TEH_DB_run_transaction (rc->connection, "run withdraw", + TEH_MT_WITHDRAW, &mhd_ret, &withdraw_transaction, &wc)) From add2d29acf1d73258c034f82a544e591009cd238 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Tue, 7 Dec 2021 21:27:08 +0100 Subject: [PATCH 04/37] implement basic /metrics --- src/exchange/taler-exchange-httpd_metrics.c | 58 +++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_metrics.c b/src/exchange/taler-exchange-httpd_metrics.c index d3f1f02d8..39e20a859 100644 --- a/src/exchange/taler-exchange-httpd_metrics.c +++ b/src/exchange/taler-exchange-httpd_metrics.c @@ -38,10 +38,62 @@ MHD_RESULT TEH_handler_metrics (struct TEH_RequestContext *rc, const char *const args[]) { + char *reply; + struct MHD_Response *resp; + MHD_RESULT ret; + (void) args; - return TALER_MHD_reply_json (rc->connection, - json_pack ("{}"), - MHD_HTTP_NO_CONTENT); + GNUNET_asprintf (&reply, + "received_requests(type='%s') %llu\n" + "serialization_failures(type='%s') %llu\n" + "received_requests(type='%s') %llu\n" + "serialization_failures(type='%s') %llu\n" + "received_requests(type='%s') %llu\n" + "serialization_failures(type='%s') %llu\n" + "received_requests(type='%s') %llu\n" + "serialization_failures(type='%s') %llu\n" + "received_requests(type='%s') %llu\n" + "serialization_failures(type='%s') %llu\n" + "received_requests(type='%s') %llu\n" + "serialization_failures(type='%s') %llu\n" + "received_requests(type='%s') %llu\n" + "serialization_failures(type='%s') %llu\n", + "other", + TEH_METRICS_num_requests[TEH_MT_OTHER], + "other", + TEH_METRICS_num_conflict[TEH_MT_OTHER], + "deposit", + TEH_METRICS_num_requests[TEH_MT_DEPOSIT], + "deposit", + TEH_METRICS_num_conflict[TEH_MT_DEPOSIT], + "withdraw", + TEH_METRICS_num_requests[TEH_MT_WITHDRAW], + "withdraw", + TEH_METRICS_num_conflict[TEH_MT_WITHDRAW], + "melt", + TEH_METRICS_num_requests[TEH_MT_MELT], + "melt", + TEH_METRICS_num_conflict[TEH_MT_MELT], + "reveal-precheck", + TEH_METRICS_num_requests[TEH_MT_REVEAL_PRECHECK], + "reveal-precheck", + TEH_METRICS_num_conflict[TEH_MT_REVEAL_PRECHECK], + "reveal", + TEH_METRICS_num_requests[TEH_MT_REVEAL], + "reveal", + TEH_METRICS_num_conflict[TEH_MT_REVEAL], + "reveal-persist", + TEH_METRICS_num_requests[TEH_MT_REVEAL_PERSIST], + "reveal-persist", + TEH_METRICS_num_conflict[TEH_MT_REVEAL_PERSIST]); + resp = MHD_create_response_from_buffer (strlen (reply), + reply, + MHD_RESPMEM_MUST_FREE); + ret = MHD_queue_response (rc->connection, + MHD_HTTP_OK, + resp); + MHD_destroy_response (resp); + return ret; } From 356c59ebb688dca23d7fa1680c0d871d41bdbe84 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 08:57:09 +0100 Subject: [PATCH 05/37] -improve graphana compatibility --- src/exchange/taler-exchange-httpd_metrics.c | 72 +++++++++++++-------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_metrics.c b/src/exchange/taler-exchange-httpd_metrics.c index 39e20a859..2a828aa3f 100644 --- a/src/exchange/taler-exchange-httpd_metrics.c +++ b/src/exchange/taler-exchange-httpd_metrics.c @@ -44,48 +44,64 @@ TEH_handler_metrics (struct TEH_RequestContext *rc, (void) args; GNUNET_asprintf (&reply, - "received_requests(type='%s') %llu\n" - "serialization_failures(type='%s') %llu\n" - "received_requests(type='%s') %llu\n" - "serialization_failures(type='%s') %llu\n" - "received_requests(type='%s') %llu\n" - "serialization_failures(type='%s') %llu\n" - "received_requests(type='%s') %llu\n" - "serialization_failures(type='%s') %llu\n" - "received_requests(type='%s') %llu\n" - "serialization_failures(type='%s') %llu\n" - "received_requests(type='%s') %llu\n" - "serialization_failures(type='%s') %llu\n" - "received_requests(type='%s') %llu\n" - "serialization_failures(type='%s') %llu\n", - "other", - TEH_METRICS_num_requests[TEH_MT_OTHER], + "# HELP taler_exchange_serialization_failures " + " number of database serialization errors by +type\n" + "# TYPE taler_exchange_serialization_failures +counter\n" + "taler_exchange_serialization_failures{type='%s'} +%llu\n" + "taler_exchange_serialization_failures{type='%s'} +%llu\n" + "taler_exchange_serialization_failures{type='%s'} +%llu\n" + "taler_exchange_serialization_failures{type='%s'} +%llu\n" + "taler_exchange_serialization_failures{type='%s'} +%llu\n" + "taler_exchange_serialization_failures{type='%s'} +%llu\n" + "taler_exchange_serialization_failures{type='%s'} +%llu\n" + "# HELP taler_exchange_received_requests " + " number of received requests by type\n" + "# TYPE taler_exchange_received_requests counter\n" + "taler_exchange_received_requests{type='%s'} %llu\n" + "taler_exchange_received_requests{type='%s'} %llu\n" + "taler_exchange_received_requests{type='%s'} %llu\n" + "taler_exchange_received_requests{type='%s'} %llu\n" + "taler_exchange_received_requests{type='%s'} %llu\n" + "taler_exchange_received_requests{type='%s'} %llu\n" + "taler_exchange_received_requests{type='%s'} +%llu\n", "other", TEH_METRICS_num_conflict[TEH_MT_OTHER], "deposit", - TEH_METRICS_num_requests[TEH_MT_DEPOSIT], - "deposit", TEH_METRICS_num_conflict[TEH_MT_DEPOSIT], "withdraw", - TEH_METRICS_num_requests[TEH_MT_WITHDRAW], - "withdraw", TEH_METRICS_num_conflict[TEH_MT_WITHDRAW], "melt", - TEH_METRICS_num_requests[TEH_MT_MELT], - "melt", TEH_METRICS_num_conflict[TEH_MT_MELT], "reveal-precheck", - TEH_METRICS_num_requests[TEH_MT_REVEAL_PRECHECK], - "reveal-precheck", TEH_METRICS_num_conflict[TEH_MT_REVEAL_PRECHECK], "reveal", - TEH_METRICS_num_requests[TEH_MT_REVEAL], - "reveal", TEH_METRICS_num_conflict[TEH_MT_REVEAL], "reveal-persist", - TEH_METRICS_num_requests[TEH_MT_REVEAL_PERSIST], + TEH_METRICS_num_conflict[TEH_MT_REVEAL_PERSIST], + "other", + TEH_METRICS_num_requests[TEH_MT_OTHER], + "deposit", + TEH_METRICS_num_requests[TEH_MT_DEPOSIT], + "withdraw", + TEH_METRICS_num_requests[TEH_MT_WITHDRAW], + "melt", + TEH_METRICS_num_requests[TEH_MT_MELT], + "reveal-precheck", + TEH_METRICS_num_requests[TEH_MT_REVEAL_PRECHECK], + "reveal", + TEH_METRICS_num_requests[TEH_MT_REVEAL], "reveal-persist", - TEH_METRICS_num_conflict[TEH_MT_REVEAL_PERSIST]); + TEH_METRICS_num_requests[TEH_MT_REVEAL_PERSIST]); resp = MHD_create_response_from_buffer (strlen (reply), reply, MHD_RESPMEM_MUST_FREE); From be9cd041509fa35cec5cb2ba2886323aa6e9173e Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 09:21:08 +0100 Subject: [PATCH 06/37] -fix ftbfs --- src/exchange/taler-exchange-httpd_metrics.c | 30 +++++++-------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_metrics.c b/src/exchange/taler-exchange-httpd_metrics.c index 2a828aa3f..df37280ca 100644 --- a/src/exchange/taler-exchange-httpd_metrics.c +++ b/src/exchange/taler-exchange-httpd_metrics.c @@ -45,24 +45,15 @@ TEH_handler_metrics (struct TEH_RequestContext *rc, (void) args; GNUNET_asprintf (&reply, "# HELP taler_exchange_serialization_failures " - " number of database serialization errors by -type\n" - "# TYPE taler_exchange_serialization_failures -counter\n" - "taler_exchange_serialization_failures{type='%s'} -%llu\n" - "taler_exchange_serialization_failures{type='%s'} -%llu\n" - "taler_exchange_serialization_failures{type='%s'} -%llu\n" - "taler_exchange_serialization_failures{type='%s'} -%llu\n" - "taler_exchange_serialization_failures{type='%s'} -%llu\n" - "taler_exchange_serialization_failures{type='%s'} -%llu\n" - "taler_exchange_serialization_failures{type='%s'} -%llu\n" + " number of database serialization errors by type\n" + "# TYPE taler_exchange_serialization_failures counter\n" + "taler_exchange_serialization_failures{type='%s'} %llu\n" + "taler_exchange_serialization_failures{type='%s'} %llu\n" + "taler_exchange_serialization_failures{type='%s'} %llu\n" + "taler_exchange_serialization_failures{type='%s'} %llu\n" + "taler_exchange_serialization_failures{type='%s'} %llu\n" + "taler_exchange_serialization_failures{type='%s'} %llu\n" + "taler_exchange_serialization_failures{type='%s'} %llu\n" "# HELP taler_exchange_received_requests " " number of received requests by type\n" "# TYPE taler_exchange_received_requests counter\n" @@ -72,8 +63,7 @@ counter\n" "taler_exchange_received_requests{type='%s'} %llu\n" "taler_exchange_received_requests{type='%s'} %llu\n" "taler_exchange_received_requests{type='%s'} %llu\n" - "taler_exchange_received_requests{type='%s'} -%llu\n", + "taler_exchange_received_requests{type='%s'} %llu\n", "other", TEH_METRICS_num_conflict[TEH_MT_OTHER], "deposit", From a7273486382c69b23356a110700665b18911d0db Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 11:53:27 +0100 Subject: [PATCH 07/37] -wip --- src/exchangedb/melt.sql | 280 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 src/exchangedb/melt.sql diff --git a/src/exchangedb/melt.sql b/src/exchangedb/melt.sql new file mode 100644 index 000000000..af1aa8d4a --- /dev/null +++ b/src/exchangedb/melt.sql @@ -0,0 +1,280 @@ + +-- Everything in one big transaction +-- BEGIN; + +-- Check patch versioning is in place. +-- SELECT _v.register_patch('exchange-000x', NULL, NULL); + +CREATE OR REPLACE FUNCTION exchange_do_melt( + IN denom_val INT8, -- value of the denomination of the coin + IN denom_frac INT4, -- value of the denomination of the coin + IN amount_val INT8, -- requested melt amount (with fee) + IN amount_frac INT4, -- requested melt amount (with fee) + IN in_rc BYTEA, -- refresh session hash + IN in_coin_pub BYTEA, -- coin public key + IN coin_sig BYTEA, -- melt signature + IN in_noreveal_index INT4, -- suggested random noreveal index + IN zombie_required BOOLEAN, -- do we need a zombie coin? + OUT out_noreval_index INT4, -- noreveal index to actually use + OUT balance_ok BOOLEAN, -- balance satisfied? + OUT zombie_ok BOOLEAN, -- zombie satisfied? + OUT melt_ok BOOLEAN) -- everything OK? +LANGUAGE plpgsql +AS $$ +DECLARE + coin_uuid INT8; -- known_coin_id of coin_pub +DECLARE + tmp_val INT8; -- temporary result +DECLARE + tmp_frac INT8; -- temporary result +DECLARE + spent_val INT8; -- how much of coin was spent? +DECLARE + spent_frac INT8; -- how much of coin was spent? +DECLARE + unspent_val INT8; -- how much of coin was refunded? +DECLARE + unspent_frac INT8; -- how much of coin was refunded? +BEGIN + +SELECT known_coin_id INTO coin_uuid + FROM known_coins + WHERE coin_pub=in_coin_pub; + +IF NOT FOUND +THEN + -- coin unknown, should be impossible! + out_noreveal_index=-1; + balance_ok=FALSE; + zombie_ok=FALSE; + melt_ok=FALSE; + ASSERT false, 'coin unknown'; + RETURN; +END IF; + +-- We optimistically insert, and then on conflict declare +-- the query successful due to idempotency. +INSERT INTO refresh_commitments + (rc + ,old_known_coin_id + ,old_coin_sig + ,amount_with_fee_val + ,amount_with_fee_frac + ,noreveal_index) +VALUES + (in_rc + ,coin_uuid + ,coin_sig + ,amount_val + ,amount_frac + ,in_noreveal_index) +ON CONFLICT DO NOTHING; + +IF FOUND +THEN + -- already melted, get noreveal_index + SELECT noreveal_index INTO out_noreveal_index + FROM refresh_commitments + WHERE rc=in_rc ; + balance_ok=TRUE; + zombie_ok=TRUE; + melt_ok=TRUE; + RETURN; +END IF; + +-- Need to check for sufficient balance... +spent_val = 0; +spent_frac = 0; +unspent_val = 0; +unspent_frac = 0; + +SELECT + SUM(amount_with_fee_val) -- overflow here is not plausible + ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM deposits + WHERE known_coin_id=coin_uuid; + +spent_val = spent_val + tmp_val; +spent_frac = spent_frac + tmp_frac; + +SELECT + SUM(amount_with_fee_val) -- overflow here is not plausible + ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM refresh_commitments + WHERE old_known_coin_id=coin_uuid; + +spent_val = spent_val + tmp_val; +spent_frac = spent_frac + tmp_frac; + +SELECT + SUM(amount_with_fee_val) -- overflow here is not plausible + ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM refunds + WHERE known_coin_id=coin_uuid; + +unspent_val = unspent_val + tmp_val; +unspent_frac = unspent_frac + tmp_frac; + +SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup_refresh + WHERE known_coin_id=coin_uuid; + +unspent_val = unspent_val + tmp_val; +unspent_frac = unspent_frac + tmp_frac; + +SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup + WHERE known_coin_id=coin_uuid; + +spent_val = spent_val + tmp_val; +spent_frac = spent_frac + tmp_frac; + +SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup_refresh + JOIN refresh_revealed_coins rrc + USING (rrc_serial) + JOIN refresh_commitments rfc + ON (rrc.melt_serial_id = rfc.melt_serial_id) + WHERE rfc.old_known_coin_id=coin_uuid; + +spent_val = spent_val + tmp_val; +spent_frac = spent_frac + tmp_frac; + + +------------------- TBD from here + +SELECT + reserve_uuid + ,current_balance_val + ,current_balance_frac_uuid + ,expiration_date + ,gc_date + INTO + reserve_uuid + ,reserve_val + ,reserve_frac + ,reserve_gc + FROM reserves + WHERE reserve_pub=reserve_pub; + +IF NOT FOUND +THEN + -- reserve unknown + reserve_found=FALSE; + balance_ok=FALSE; + kyc_ok=FALSE; + RETURN; +END IF; + +-- We optimistically insert, and then on conflict declare +-- the query successful due to idempotency. +INSERT INTO reserves_out + (h_blind_ev + ,denom_serial + ,denom_sig + ,reserve_uuid + ,reserve_sig + ,execution_date + ,amount_with_fee_val + ,amount_with_fee_frac) +VALUES + (h_coin_envelope + ,denom_serial + ,denom_sig + ,reserve_uuid + ,reserve_sig + ,now + ,amount_val + ,amount_frac) +ON CONFLICT DO NOTHING; + +IF NOT FOUND +THEN + -- idempotent query, all constraints must be satisfied + reserve_found=TRUE; + balance_ok=TRUE; + kyc_ok=TRUE; + RETURN; +END IF; + +-- Check reserve balance is sufficient. +IF (reserve_val > amount_val) +THEN + IF (reserve_frac > amount_frac) + THEN + reserve_val=reserve_val - amount_val; + reserve_frac=reserve_frac - amount_frac; + ELSE + reserve_val=reserve_val - amount_val - 1; + reserve_frac=reserve_frac + 100000000 - amount_frac; + END IF; +ELSE + IF (reserve_val == amount_val) AND (reserve_frac >= amount_frac) + THEN + reserve_val=0; + reserve_frac=reserve_frac - amount_frac; + ELSE + reserve_found=TRUE; + balance_ok=FALSE; + kyc_ok=FALSE; -- we do not really know or care + RETURN; + END IF; +END IF; + +-- Calculate new expiration dates. +min_reserve_gc=MAX(min_reserve_gc,reserve_gc); + +-- Update reserve balance. +UPDATE reserves SET + gc_date=min_reserve_gc + ,current_balance_val=reserve_val + ,current_balance_frac=reserve_frac +WHERE + reserve_uuid=reserve_uuid; + +reserve_found=TRUE; +balance_ok=TRUE; + +-- Obtain KYC status based on the last wire transfer into +-- this reserve. FIXME: likely not adequate for reserves that got P2P transfers! +SELECT kyc_ok + INTO kyc_ok + FROM reserves_in + JOIN wire_targets USING (wire_target_serial_id) + WHERE reserve_uuid=reserve_uuid + LIMIT 1; -- limit 1 should not be required (without p2p transfers) + + + +END $$; + +COMMENT ON FUNCTION exchange_do_melt(INT8, INT4, BYTEA, BYTEA, BYTEA, BYTEA, BYTEA, INT8, INT8) + IS 'Checks whether the coin has sufficient balance for a melt operation (or the request is repeated and was previously approved) and if so updates the database with the result'; + + +-- Complete transaction +-- COMMIT; From 98e44f33a766f049085d7f76d2a11a3ad1d91d57 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 12:42:22 +0100 Subject: [PATCH 08/37] add missing preflight --- src/exchange/taler-exchange-httpd_keys.c | 2 ++ src/util/taler-exchange-secmod-rsa.c | 13 +++++++++++++ src/util/test_helper_eddsa.conf | 1 - src/util/test_helper_rsa.c | 2 ++ src/util/test_helper_rsa.conf | 2 -- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_keys.c b/src/exchange/taler-exchange-httpd_keys.c index f722c16a7..587e0eed9 100644 --- a/src/exchange/taler-exchange-httpd_keys.c +++ b/src/exchange/taler-exchange-httpd_keys.c @@ -1935,6 +1935,8 @@ build_key_state (struct HelperState *hs, ksh->auditors = json_array (); GNUNET_assert (NULL != ksh->auditors); /* NOTE: fetches master-signed signkeys, but ALSO those that were revoked! */ + GNUNET_break (GNUNET_OK == + TEH_plugin->preflight (TEH_plugin->cls)); qs = TEH_plugin->iterate_denominations (TEH_plugin->cls, &denomination_info_cb, ksh); diff --git a/src/util/taler-exchange-secmod-rsa.c b/src/util/taler-exchange-secmod-rsa.c index 3b06a56ed..e121e9ab6 100644 --- a/src/util/taler-exchange-secmod-rsa.c +++ b/src/util/taler-exchange-secmod-rsa.c @@ -868,6 +868,19 @@ update_keys (struct Denomination *denom, bool *wake) { /* create new denomination keys */ + if (NULL != denom->keys_tail) + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Updating keys of denomination `%s', last key %s valid for another %s\n", + denom->section, + GNUNET_h2s (&denom->keys_tail->h_rsa.hash), + GNUNET_STRINGS_relative_time_to_string ( + GNUNET_TIME_absolute_get_remaining ( + GNUNET_TIME_absolute_subtract ( + GNUNET_TIME_absolute_add ( + denom->keys_tail->anchor, + denom->duration_withdraw), + overlap_duration)), + GNUNET_YES)); while ( (NULL == denom->keys_tail) || GNUNET_TIME_absolute_is_past ( GNUNET_TIME_absolute_subtract ( diff --git a/src/util/test_helper_eddsa.conf b/src/util/test_helper_eddsa.conf index 8fe119c40..a13833c02 100644 --- a/src/util/test_helper_eddsa.conf +++ b/src/util/test_helper_eddsa.conf @@ -1,7 +1,6 @@ [PATHS] # Persistent data storage for the testcase TALER_TEST_HOME = test_helper_eddsa_home/ -TALER_RUNTIME_DIR = ${TMPDIR:-/tmp}/${USER}/test_helper_eddsa/ [taler-exchange-secmod-eddsa] CLIENT_DIR = $TALER_RUNTIME_DIR diff --git a/src/util/test_helper_rsa.c b/src/util/test_helper_rsa.c index 14ff2bfab..bafa45ba8 100644 --- a/src/util/test_helper_rsa.c +++ b/src/util/test_helper_rsa.c @@ -609,6 +609,8 @@ main (int argc, (void) argc; (void) argv; + unsetenv ("XDG_DATA_HOME"); + unsetenv ("XDG_CONFIG_HOME"); GNUNET_log_setup ("test-helper-rsa", "WARNING", NULL); diff --git a/src/util/test_helper_rsa.conf b/src/util/test_helper_rsa.conf index 66127ee01..6f445fc56 100644 --- a/src/util/test_helper_rsa.conf +++ b/src/util/test_helper_rsa.conf @@ -1,8 +1,6 @@ [PATHS] # Persistent data storage for the testcase TALER_TEST_HOME = test_helper_rsa_home/ -TALER_RUNTIME_DIR = ${TMPDIR:-/tmp}/${USER}/test_helper_rsa/ - [coin_1] DURATION_WITHDRAW = 1 minute From c1aaafca91ba60082ba3fff969b0c531613cfe50 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 14:32:54 +0100 Subject: [PATCH 09/37] -fix syntax --- src/exchange/taler-exchange-httpd_metrics.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_metrics.c b/src/exchange/taler-exchange-httpd_metrics.c index df37280ca..c7165326b 100644 --- a/src/exchange/taler-exchange-httpd_metrics.c +++ b/src/exchange/taler-exchange-httpd_metrics.c @@ -72,11 +72,11 @@ TEH_handler_metrics (struct TEH_RequestContext *rc, TEH_METRICS_num_conflict[TEH_MT_WITHDRAW], "melt", TEH_METRICS_num_conflict[TEH_MT_MELT], - "reveal-precheck", + "reveal_precheck", TEH_METRICS_num_conflict[TEH_MT_REVEAL_PRECHECK], "reveal", TEH_METRICS_num_conflict[TEH_MT_REVEAL], - "reveal-persist", + "reveal_persist", TEH_METRICS_num_conflict[TEH_MT_REVEAL_PERSIST], "other", TEH_METRICS_num_requests[TEH_MT_OTHER], @@ -86,11 +86,11 @@ TEH_handler_metrics (struct TEH_RequestContext *rc, TEH_METRICS_num_requests[TEH_MT_WITHDRAW], "melt", TEH_METRICS_num_requests[TEH_MT_MELT], - "reveal-precheck", + "reveal_precheck", TEH_METRICS_num_requests[TEH_MT_REVEAL_PRECHECK], "reveal", TEH_METRICS_num_requests[TEH_MT_REVEAL], - "reveal-persist", + "reveal_persist", TEH_METRICS_num_requests[TEH_MT_REVEAL_PERSIST]); resp = MHD_create_response_from_buffer (strlen (reply), reply, From 32e185978fb96f9099c97caf45074c1a605c08c6 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 14:33:13 +0100 Subject: [PATCH 10/37] tolerate missing 'Expire' header better --- src/lib/exchange_api_handle.c | 51 ++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/src/lib/exchange_api_handle.c b/src/lib/exchange_api_handle.c index 7103f6d6d..07e124880 100644 --- a/src/lib/exchange_api_handle.c +++ b/src/lib/exchange_api_handle.c @@ -52,6 +52,17 @@ */ #define EXCHANGE_SERIALIZATION_FORMAT_VERSION 0 +/** + * How far off do we allow key liftimes to be? + */ +#define LIFETIME_TOLERANCE GNUNET_TIME_UNIT_HOURS + +/** + * If the "Expire" cache control header is missing, for + * how long do we assume the reply to be valid at least? + */ +#define DEFAULT_EXPIRATION GNUNET_TIME_UNIT_HOURS + /** * Set to 1 for extra debug logging. */ @@ -847,7 +858,7 @@ decode_keys_json (const json_t *resp_obj, key_data->denom_keys[key_data->num_denom_keys++] = dk; /* Update "last_denom_issue_date" */ - TALER_LOG_DEBUG ("Adding denomination key that is valid_from %s\n", + TALER_LOG_DEBUG ("Adding denomination key that is valid_until %s\n", GNUNET_STRINGS_absolute_time_to_string (dk.valid_from)); key_data->last_denom_issue_date = GNUNET_TIME_absolute_max (key_data->last_denom_issue_date, @@ -1053,6 +1064,9 @@ void TALER_EXCHANGE_set_last_denom (struct TALER_EXCHANGE_Handle *exchange, struct GNUNET_TIME_Absolute last_denom_new) { + TALER_LOG_DEBUG ( + "Application explicitly set last denomination validity to %s\n", + GNUNET_STRINGS_absolute_time_to_string (last_denom_new)); exchange->key_data.last_denom_issue_date = last_denom_new; } @@ -1117,9 +1131,19 @@ keys_completed_cb (void *cls, }; GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Received keys from URL `%s' with status %ld.\n", + "Received keys from URL `%s' with status %ld and expiration %s.\n", kr->url, - response_code); + response_code, + GNUNET_STRINGS_absolute_time_to_string (kr->expire)); + if (GNUNET_TIME_absolute_is_past (kr->expire)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Exchange failed to give expiration time, assuming in %s\n", + GNUNET_STRINGS_relative_time_to_string (DEFAULT_EXPIRATION, + GNUNET_YES)); + kr->expire = GNUNET_TIME_relative_to_absolute (DEFAULT_EXPIRATION); + (void) GNUNET_TIME_round_abs (&kr->expire); + } kd_old = exchange->key_data; memset (&kd, 0, @@ -1253,9 +1277,14 @@ keys_completed_cb (void *cls, break; } exchange->key_data = kd; - TALER_LOG_DEBUG ("Last DK issue date update to: %s\n", - GNUNET_STRINGS_absolute_time_to_string - (exchange->key_data.last_denom_issue_date)); + if (GNUNET_TIME_absolute_is_past (exchange->key_data.last_denom_issue_date)) + TALER_LOG_WARNING ("Last DK issue date from exchange is in the past: %s\n", + GNUNET_STRINGS_absolute_time_to_string ( + exchange->key_data.last_denom_issue_date)); + else + TALER_LOG_DEBUG ("Last DK issue date updated to: %s\n", + GNUNET_STRINGS_absolute_time_to_string ( + exchange->key_data.last_denom_issue_date)); if (MHD_HTTP_OK != response_code) @@ -2010,10 +2039,12 @@ TALER_EXCHANGE_test_signing_key (const struct TALER_EXCHANGE_Keys *keys, /* we will check using a tolerance of 1h for the time */ now = GNUNET_TIME_absolute_get (); for (unsigned int i = 0; inum_sign_keys; i++) - if ( (keys->sign_keys[i].valid_from.abs_value_us <= now.abs_value_us + 60 - * 60 * 1000LL * 1000LL) && - (keys->sign_keys[i].valid_until.abs_value_us > now.abs_value_us - 60 - * 60 * 1000LL * 1000LL) && + if ( (keys->sign_keys[i].valid_from.abs_value_us <= + GNUNET_TIME_absolute_add (now, + LIFETIME_TOLERANCE).abs_value_us) && + (keys->sign_keys[i].valid_until.abs_value_us > + GNUNET_TIME_absolute_subtract (now, + LIFETIME_TOLERANCE).abs_value_us) && (0 == GNUNET_memcmp (pub, &keys->sign_keys[i].key)) ) return GNUNET_OK; From 5d765a294a06fa6a1665f6e9736b839d3e92ceba Mon Sep 17 00:00:00 2001 From: Boss Marco Date: Wed, 8 Dec 2021 14:43:12 +0100 Subject: [PATCH 11/37] fix prometheus metric labels --- src/exchange/taler-exchange-httpd_metrics.c | 36 ++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_metrics.c b/src/exchange/taler-exchange-httpd_metrics.c index c7165326b..2ea889ff0 100644 --- a/src/exchange/taler-exchange-httpd_metrics.c +++ b/src/exchange/taler-exchange-httpd_metrics.c @@ -47,23 +47,23 @@ TEH_handler_metrics (struct TEH_RequestContext *rc, "# HELP taler_exchange_serialization_failures " " number of database serialization errors by type\n" "# TYPE taler_exchange_serialization_failures counter\n" - "taler_exchange_serialization_failures{type='%s'} %llu\n" - "taler_exchange_serialization_failures{type='%s'} %llu\n" - "taler_exchange_serialization_failures{type='%s'} %llu\n" - "taler_exchange_serialization_failures{type='%s'} %llu\n" - "taler_exchange_serialization_failures{type='%s'} %llu\n" - "taler_exchange_serialization_failures{type='%s'} %llu\n" - "taler_exchange_serialization_failures{type='%s'} %llu\n" + "taler_exchange_serialization_failures{type=\"%s\"} %llu\n" + "taler_exchange_serialization_failures{type=\"%s\"} %llu\n" + "taler_exchange_serialization_failures{type=\"%s\"} %llu\n" + "taler_exchange_serialization_failures{type=\"%s\"} %llu\n" + "taler_exchange_serialization_failures{type=\"%s\"} %llu\n" + "taler_exchange_serialization_failures{type=\"%s\"} %llu\n" + "taler_exchange_serialization_failures{type=\"%s\"} %llu\n" "# HELP taler_exchange_received_requests " " number of received requests by type\n" "# TYPE taler_exchange_received_requests counter\n" - "taler_exchange_received_requests{type='%s'} %llu\n" - "taler_exchange_received_requests{type='%s'} %llu\n" - "taler_exchange_received_requests{type='%s'} %llu\n" - "taler_exchange_received_requests{type='%s'} %llu\n" - "taler_exchange_received_requests{type='%s'} %llu\n" - "taler_exchange_received_requests{type='%s'} %llu\n" - "taler_exchange_received_requests{type='%s'} %llu\n", + "taler_exchange_received_requests{type=\"%s\"} %llu\n" + "taler_exchange_received_requests{type=\"%s\"} %llu\n" + "taler_exchange_received_requests{type=\"%s\"} %llu\n" + "taler_exchange_received_requests{type=\"%s\"} %llu\n" + "taler_exchange_received_requests{type=\"%s\"} %llu\n" + "taler_exchange_received_requests{type=\"%s\"} %llu\n" + "taler_exchange_received_requests{type=\"%s\"} %llu\n", "other", TEH_METRICS_num_conflict[TEH_MT_OTHER], "deposit", @@ -72,11 +72,11 @@ TEH_handler_metrics (struct TEH_RequestContext *rc, TEH_METRICS_num_conflict[TEH_MT_WITHDRAW], "melt", TEH_METRICS_num_conflict[TEH_MT_MELT], - "reveal_precheck", + "reveal-precheck", TEH_METRICS_num_conflict[TEH_MT_REVEAL_PRECHECK], "reveal", TEH_METRICS_num_conflict[TEH_MT_REVEAL], - "reveal_persist", + "reveal-persist", TEH_METRICS_num_conflict[TEH_MT_REVEAL_PERSIST], "other", TEH_METRICS_num_requests[TEH_MT_OTHER], @@ -86,11 +86,11 @@ TEH_handler_metrics (struct TEH_RequestContext *rc, TEH_METRICS_num_requests[TEH_MT_WITHDRAW], "melt", TEH_METRICS_num_requests[TEH_MT_MELT], - "reveal_precheck", + "reveal-precheck", TEH_METRICS_num_requests[TEH_MT_REVEAL_PRECHECK], "reveal", TEH_METRICS_num_requests[TEH_MT_REVEAL], - "reveal_persist", + "reveal-persist", TEH_METRICS_num_requests[TEH_MT_REVEAL_PERSIST]); resp = MHD_create_response_from_buffer (strlen (reply), reply, From 2e6e2387b8c4e83e32c50e4fc8cb5312bce68735 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 14:53:20 +0100 Subject: [PATCH 12/37] logging --- src/lib/exchange_api_handle.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/exchange_api_handle.c b/src/lib/exchange_api_handle.c index 07e124880..49e39a0e9 100644 --- a/src/lib/exchange_api_handle.c +++ b/src/lib/exchange_api_handle.c @@ -1152,6 +1152,9 @@ keys_completed_cb (void *cls, switch (response_code) { case 0: + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Failed to receive /keys response from exchange %s\n", + exchange->url); free_keys_request (kr); exchange->keys_error_count++; exchange->kr = NULL; From 9d3f7d3a56712d6a963e8a4775a8f9757740d648 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 15:18:40 +0100 Subject: [PATCH 13/37] move ensure_coin_known for melt and recoup outside of transaction, more logging --- src/exchange/taler-exchange-httpd_deposit.c | 32 ++++++++++++++++----- src/exchange/taler-exchange-httpd_melt.c | 10 +++++++ src/exchange/taler-exchange-httpd_recoup.c | 22 +++++++++----- src/exchangedb/plugin_exchangedb_postgres.c | 6 +++- 4 files changed, 55 insertions(+), 15 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_deposit.c b/src/exchange/taler-exchange-httpd_deposit.c index 5534c402e..d549a1fa6 100644 --- a/src/exchange/taler-exchange-httpd_deposit.c +++ b/src/exchange/taler-exchange-httpd_deposit.c @@ -162,13 +162,6 @@ deposit_transaction (void *cls, enum GNUNET_DB_QueryStatus qs; struct TALER_Amount deposit_fee; - /* make sure coin is 'known' in database */ - qs = TEH_make_coin_known (&deposit->coin, - connection, - mhd_ret); - if (qs < 0) - return qs; - /* Check for idempotency: did we get this request before? */ qs = TEH_plugin->have_deposit (TEH_plugin->cls, deposit, @@ -490,6 +483,31 @@ TEH_handler_deposit (struct MHD_Connection *connection, NULL); } + if (GNUNET_SYSERR == + TEH_plugin->preflight (TEH_plugin->cls)) + { + GNUNET_break (0); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_START_FAILED, + "preflight failure"); + } + + { + MHD_RESULT mhd_ret = MHD_NO; + enum GNUNET_DB_QueryStatus qs; + + /* make sure coin is 'known' in database */ + qs = TEH_make_coin_known (&deposit.coin, + connection, + &mhd_ret); + /* no transaction => no serialization failures should be possible */ + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs); + if (qs < 0) + return mhd_ret; + } + + /* execute transaction */ { MHD_RESULT mhd_ret; diff --git a/src/exchange/taler-exchange-httpd_melt.c b/src/exchange/taler-exchange-httpd_melt.c index e2e343711..3347790b5 100644 --- a/src/exchange/taler-exchange-httpd_melt.c +++ b/src/exchange/taler-exchange-httpd_melt.c @@ -380,6 +380,16 @@ static MHD_RESULT handle_melt (struct MHD_Connection *connection, struct MeltContext *rmc) { + if (GNUNET_SYSERR == + TEH_plugin->preflight (TEH_plugin->cls)) + { + GNUNET_break (0); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_START_FAILED, + "preflight failure"); + } + /* verify signature of coin for melt operation */ { struct TALER_RefreshMeltCoinAffirmationPS body = { diff --git a/src/exchange/taler-exchange-httpd_recoup.c b/src/exchange/taler-exchange-httpd_recoup.c index 7ea345b87..fd8baf65d 100644 --- a/src/exchange/taler-exchange-httpd_recoup.c +++ b/src/exchange/taler-exchange-httpd_recoup.c @@ -130,13 +130,6 @@ recoup_transaction (void *cls, enum GNUNET_DB_QueryStatus qs; int existing_recoup_found; - /* make sure coin is 'known' in database */ - qs = TEH_make_coin_known (pc->coin, - connection, - mhd_ret); - if (qs < 0) - return qs; - /* Check whether a recoup is allowed, and if so, to which reserve / account the money should go */ if (pc->refreshed) @@ -471,6 +464,21 @@ verify_and_execute_recoup ( pc.coin_bks = coin_bks; pc.coin = coin; pc.refreshed = refreshed; + + { + MHD_RESULT mhd_ret = MHD_NO; + enum GNUNET_DB_QueryStatus qs; + + /* make sure coin is 'known' in database */ + qs = TEH_make_coin_known (coin, + connection, + &mhd_ret); + /* no transaction => no serialization failures should be possible */ + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs); + if (qs < 0) + return mhd_ret; + } + { MHD_RESULT mhd_ret; diff --git a/src/exchangedb/plugin_exchangedb_postgres.c b/src/exchangedb/plugin_exchangedb_postgres.c index 79013179a..4f0bc243d 100644 --- a/src/exchangedb/plugin_exchangedb_postgres.c +++ b/src/exchangedb/plugin_exchangedb_postgres.c @@ -5778,6 +5778,8 @@ postgres_ensure_coin_known (void *cls, GNUNET_break (0); return TALER_EXCHANGEDB_CKS_HARD_FAIL; case GNUNET_DB_STATUS_SOFT_ERROR: + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Serialization failure in insert_known_coin? Strange!\n"); return TALER_EXCHANGEDB_CKS_SOFT_FAIL; case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: /* continued below */ @@ -5794,8 +5796,11 @@ postgres_ensure_coin_known (void *cls, switch (qs) { case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); return TALER_EXCHANGEDB_CKS_HARD_FAIL; case GNUNET_DB_STATUS_SOFT_ERROR: + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Serialization failure in get_known_coin_dh? Strange!\n"); return TALER_EXCHANGEDB_CKS_SOFT_FAIL; case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: if (0 == GNUNET_memcmp (&denom_pub_hash, @@ -5865,7 +5870,6 @@ postgres_insert_deposit (void *cls, &kyc); if (qs <= 0) { - GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); return qs; } From a00cebcced1bdee1da998177c06986c1886eb55a Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Wed, 8 Dec 2021 15:54:48 +0100 Subject: [PATCH 14/37] put crypto worker in exchange-tools, re-enable build --- src/exchange-tools/.gitignore | 1 + src/exchange-tools/Makefile.am | 17 ++++++++++++++++- .../taler-crypto-worker.c | 1 - src/util/.gitignore | 1 - src/util/Makefile.am | 12 ------------ 5 files changed, 17 insertions(+), 15 deletions(-) rename src/{util => exchange-tools}/taler-crypto-worker.c (99%) diff --git a/src/exchange-tools/.gitignore b/src/exchange-tools/.gitignore index bf45e9670..69279d792 100644 --- a/src/exchange-tools/.gitignore +++ b/src/exchange-tools/.gitignore @@ -1,2 +1,3 @@ taler-exchange-offline taler-auditor-offline +taler-crypto-worker diff --git a/src/exchange-tools/Makefile.am b/src/exchange-tools/Makefile.am index 82f4cbeb9..846ea6132 100644 --- a/src/exchange-tools/Makefile.am +++ b/src/exchange-tools/Makefile.am @@ -15,7 +15,8 @@ endif bin_PROGRAMS = \ taler-auditor-offline \ taler-exchange-offline \ - taler-exchange-dbinit + taler-exchange-dbinit \ + taler-crypto-worker taler_exchange_offline_SOURCES = \ taler-exchange-offline.c @@ -59,6 +60,20 @@ taler_exchange_dbinit_CPPFLAGS = \ -I$(top_srcdir)/src/pq/ \ $(POSTGRESQL_CPPFLAGS) +taler_crypto_worker_SOURCES = \ + taler-crypto-worker.c +taler_crypto_worker_LDADD = \ + $(top_builddir)/src/util/libtalerutil.la \ + $(top_builddir)/src/json/libtalerjson.la \ + -lgnunetutil \ + -lgnunetjson \ + -ljansson \ + -lpthread \ + $(LIBGCRYPT_LIBS) \ + $(XLIB) + + + # Testcases diff --git a/src/util/taler-crypto-worker.c b/src/exchange-tools/taler-crypto-worker.c similarity index 99% rename from src/util/taler-crypto-worker.c rename to src/exchange-tools/taler-crypto-worker.c index 9c49ea374..7ca08b4b6 100644 --- a/src/util/taler-crypto-worker.c +++ b/src/exchange-tools/taler-crypto-worker.c @@ -25,7 +25,6 @@ #include "taler_error_codes.h" #include "taler_json_lib.h" #include "taler_signatures.h" -#include "secmod_common.h" /** diff --git a/src/util/.gitignore b/src/util/.gitignore index 946924dcc..f25567f32 100644 --- a/src/util/.gitignore +++ b/src/util/.gitignore @@ -1,6 +1,5 @@ taler-config test_payto -taler-crypto-worker taler-exchange-secmod-rsa taler-exchange-secmod-eddsa test_helper_rsa diff --git a/src/util/Makefile.am b/src/util/Makefile.am index 790bba735..6c64d77b2 100644 --- a/src/util/Makefile.am +++ b/src/util/Makefile.am @@ -58,18 +58,6 @@ taler_exchange_secmod_eddsa_LDADD = \ $(LIBGCRYPT_LIBS) \ $(XLIB) -taler_crypto_worker_SOURCES = \ - taler-crypto-worker.c -taler_crypto_worker_LDADD = \ - libtalerutil.la \ - -lgnunetutil \ - -lgnunetjson \ - -ljansson \ - -lpthread \ - $(LIBGCRYPT_LIBS) \ - $(XLIB) - - lib_LTLIBRARIES = \ libtalerutil.la From ec4c531cc6bd96bb3c2f6e83ac07d10a017986fb Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Wed, 8 Dec 2021 16:01:57 +0100 Subject: [PATCH 15/37] crypto worker: fix refresh planchet setup --- src/exchange-tools/taler-crypto-worker.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/exchange-tools/taler-crypto-worker.c b/src/exchange-tools/taler-crypto-worker.c index 7ca08b4b6..d96bfe948 100644 --- a/src/exchange-tools/taler-crypto-worker.c +++ b/src/exchange-tools/taler-crypto-worker.c @@ -144,6 +144,7 @@ run (void *cls, struct TALER_Amount value; struct TALER_ReservePublicKeyP reserve_pub; struct TALER_ReservePublicKeyP reserve_priv; + struct TALER_TransferSecretP transfer_secret; uint32_t coin_index; json_t *resp; struct GNUNET_JSON_Specification eddsa_verify_spec[] = { @@ -159,6 +160,8 @@ run (void *cls, &reserve_priv), GNUNET_JSON_spec_uint32 ("coin_index", &coin_index), + GNUNET_JSON_spec_fixed_auto ("transfer_secret", + &transfer_secret), GNUNET_JSON_spec_end () }; struct TALER_CoinSpendPublicKeyP coin_pub; @@ -175,11 +178,9 @@ run (void *cls, global_ret = 1; return; } -#if FIXME_FLORIAN TALER_planchet_setup_refresh (&transfer_secret, - coin_num_salt, + coin_index, &ps); -#endif GNUNET_CRYPTO_eddsa_key_get_public (&ps.coin_priv.eddsa_priv, &coin_pub.eddsa_pub); From 54177c6e1ddc63cb3411e42ee56fbba143c71614 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Wed, 8 Dec 2021 16:05:21 +0100 Subject: [PATCH 16/37] crypto worker: remove unused args --- src/exchange-tools/taler-crypto-worker.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/exchange-tools/taler-crypto-worker.c b/src/exchange-tools/taler-crypto-worker.c index d96bfe948..674724eda 100644 --- a/src/exchange-tools/taler-crypto-worker.c +++ b/src/exchange-tools/taler-crypto-worker.c @@ -139,25 +139,10 @@ run (void *cls, } if (0 == strcmp ("setup_refresh_planchet", op)) { - struct TALER_DenominationPublicKey denom_pub; - struct TALER_Amount fee_withdraw; - struct TALER_Amount value; - struct TALER_ReservePublicKeyP reserve_pub; - struct TALER_ReservePublicKeyP reserve_priv; struct TALER_TransferSecretP transfer_secret; uint32_t coin_index; json_t *resp; struct GNUNET_JSON_Specification eddsa_verify_spec[] = { - TALER_JSON_spec_denom_pub ("denom_pub", - &denom_pub), - TALER_JSON_spec_amount_any ("fee_withdraw", - &fee_withdraw), - TALER_JSON_spec_amount_any ("value", - &value), - GNUNET_JSON_spec_fixed_auto ("reserve_pub", - &reserve_pub), - GNUNET_JSON_spec_fixed_auto ("reserve_priv", - &reserve_priv), GNUNET_JSON_spec_uint32 ("coin_index", &coin_index), GNUNET_JSON_spec_fixed_auto ("transfer_secret", From b64f718037aa318e83424fa6e2e83021908ff1dc Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 16:11:27 +0100 Subject: [PATCH 17/37] more aggressive rollback on failures --- src/exchange/taler-exchange-httpd_db.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_db.c b/src/exchange/taler-exchange-httpd_db.c index da495517a..c091f994d 100644 --- a/src/exchange/taler-exchange-httpd_db.c +++ b/src/exchange/taler-exchange-httpd_db.c @@ -163,15 +163,20 @@ TEH_DB_run_transaction (struct MHD_Connection *connection, if (GNUNET_DB_STATUS_HARD_ERROR == qs) return GNUNET_SYSERR; if (0 <= qs) - qs = TEH_plugin->commit (TEH_plugin->cls); - if (GNUNET_DB_STATUS_HARD_ERROR == qs) { - if (NULL != mhd_ret) - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_COMMIT_FAILED, - NULL); - return GNUNET_SYSERR; + qs = TEH_plugin->commit (TEH_plugin->cls); + if (GNUNET_DB_STATUS_HARD_ERROR == qs) + { + TEH_plugin->rollback (TEH_plugin->cls); + if (NULL != mhd_ret) + *mhd_ret = TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_COMMIT_FAILED, + NULL); + return GNUNET_SYSERR; + } + if (0 > qs) + TEH_plugin->rollback (TEH_plugin->cls); } /* make sure callback did not violate invariants! */ GNUNET_assert ( (NULL == mhd_ret) || From ed7379d235b39d71e4cd94f90b52627c32c6d8e7 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 16:11:46 +0100 Subject: [PATCH 18/37] consider key rotation frequency instead of earliest expiration for Expire header --- src/exchange/taler-exchange-httpd_keys.c | 56 ++++++++++++++---------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_keys.c b/src/exchange/taler-exchange-httpd_keys.c index 587e0eed9..fbca5d652 100644 --- a/src/exchange/taler-exchange-httpd_keys.c +++ b/src/exchange/taler-exchange-httpd_keys.c @@ -322,9 +322,10 @@ struct TEH_KeyStateHandle struct GNUNET_TIME_Absolute reload_time; /** - * When is the next key invalid and we expect to have a different reply? + * What is the period at which we rotate keys + * (signing or denomination keys)? */ - struct GNUNET_TIME_Absolute next_reload; + struct GNUNET_TIME_Relative rekey_frequency; /** * When does our online signing key expire and we @@ -1370,9 +1371,9 @@ auditor_denom_cb ( struct SignKeyCtx { /** - * When does the next signing key expire. Updated. + * What is the current rotation frequency for signing keys. Updated. */ - struct GNUNET_TIME_Absolute next_sk_expire; + struct GNUNET_TIME_Relative min_sk_frequency; /** * JSON array of signing keys (being created). @@ -1399,10 +1400,14 @@ add_sign_key_cb (void *cls, struct SigningKey *sk = value; (void) pid; - ctx->next_sk_expire = - GNUNET_TIME_absolute_min (ctx->next_sk_expire, - sk->meta.expire_sign); - + if (GNUNET_TIME_absolute_is_future (sk->meta.expire_sign)) + { + ctx->min_sk_frequency = + GNUNET_TIME_relative_min (ctx->min_sk_frequency, + GNUNET_TIME_absolute_get_difference ( + sk->meta.start, + sk->meta.expire_sign)); + } GNUNET_assert ( 0 == json_array_append_new ( @@ -1438,9 +1443,10 @@ struct DenomKeyCtx json_t *recoup; /** - * When does the next denomination key expire. Updated. + * What is the minimum key rotation frequency of + * valid denomination keys? */ - struct GNUNET_TIME_Absolute next_dk_expire; + struct GNUNET_TIME_Relative min_dk_frequency; }; @@ -1475,9 +1481,14 @@ add_denom_key_cb (void *cls, } else { - dkc->next_dk_expire = - GNUNET_TIME_absolute_min (dkc->next_dk_expire, - dk->meta.expire_withdraw); + if (GNUNET_TIME_absolute_is_future (dk->meta.start)) + { + dkc->min_dk_frequency = + GNUNET_TIME_relative_min (dkc->min_dk_frequency, + GNUNET_TIME_absolute_get_difference ( + dk->meta.start, + dk->meta.expire_withdraw)); + } (void) GNUNET_CONTAINER_heap_insert (dkc->heap, dk, dk->meta.start.abs_value_us); @@ -1562,13 +1573,14 @@ setup_general_response_headers (const struct TEH_KeyStateHandle *ksh, MHD_add_response_header (response, MHD_HTTP_HEADER_LAST_MODIFIED, dat)); - if (0 != ksh->next_reload.abs_value_us) + if (! GNUNET_TIME_relative_is_zero (ksh->rekey_frequency)) { + struct GNUNET_TIME_Relative r; struct GNUNET_TIME_Absolute m; - m = GNUNET_TIME_relative_to_absolute (TEH_max_keys_caching); - m = GNUNET_TIME_absolute_min (m, - ksh->next_reload); + r = GNUNET_TIME_relative_min (TEH_max_keys_caching, + ksh->rekey_frequency); + m = GNUNET_TIME_relative_to_absolute (r); get_date_string (m, dat); GNUNET_log (GNUNET_ERROR_TYPE_INFO, @@ -1759,7 +1771,7 @@ finish_keys_response (struct TEH_KeyStateHandle *ksh) sctx.signkeys = json_array (); GNUNET_assert (NULL != sctx.signkeys); - sctx.next_sk_expire = GNUNET_TIME_UNIT_FOREVER_ABS; + sctx.min_sk_frequency = GNUNET_TIME_UNIT_FOREVER_REL; GNUNET_CONTAINER_multipeermap_iterate (ksh->signkey_map, &add_sign_key_cb, &sctx); @@ -1770,15 +1782,15 @@ finish_keys_response (struct TEH_KeyStateHandle *ksh) struct DenomKeyCtx dkc = { .recoup = recoup, .heap = heap, - .next_dk_expire = GNUNET_TIME_UNIT_FOREVER_ABS, + .min_dk_frequency = GNUNET_TIME_UNIT_FOREVER_REL, }; GNUNET_CONTAINER_multihashmap_iterate (ksh->denomkey_map, &add_denom_key_cb, &dkc); - ksh->next_reload - = GNUNET_TIME_absolute_min (dkc.next_dk_expire, - sctx.next_sk_expire); + ksh->rekey_frequency + = GNUNET_TIME_relative_min (dkc.min_dk_frequency, + sctx.min_sk_frequency); } denoms = json_array (); GNUNET_assert (NULL != denoms); From 7e84b5570adeaa027f8c7861caf6af7943edcd0b Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 18:12:28 +0100 Subject: [PATCH 19/37] fix error handling for very large uploads, fix re-generation of /keys response after Expires expires --- src/exchange/taler-exchange-httpd.c | 30 +++++++++++++++++++ src/exchange/taler-exchange-httpd_keys.c | 5 +++- ...aler-exchange-httpd_management_post_keys.c | 6 ++++ src/include/taler_mhd_lib.h | 6 ++++ src/lib/exchange_api_management_post_keys.c | 4 +++ src/mhd/mhd_parsing.c | 14 +++------ src/mhd/mhd_responses.c | 22 +++----------- 7 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/exchange/taler-exchange-httpd.c b/src/exchange/taler-exchange-httpd.c index 57c965189..58e9b572a 100644 --- a/src/exchange/taler-exchange-httpd.c +++ b/src/exchange/taler-exchange-httpd.c @@ -1003,6 +1003,36 @@ handle_mhd_request (void *cls, "illegal incoming correlation ID\n"); correlation_id = NULL; } + + /* Check if upload is in bounds */ + if (0 == strcasecmp (method, + MHD_HTTP_METHOD_POST)) + { + const char *cl; + + /* Maybe check for maximum upload size + and refuse requests if they are just too big. */ + cl = MHD_lookup_connection_value (connection, + MHD_HEADER_KIND, + MHD_HTTP_HEADER_CONTENT_LENGTH); + if (NULL != cl) + { + unsigned long long cv; + char dummy; + + if (1 != sscanf (cl, + "%llu%c", + &cv, + &dummy)) + { + /* Not valid HTTP request, just close connection. */ + GNUNET_break_op (0); + return MHD_NO; + } + if (cv > TALER_MHD_REQUEST_BUFFER_MAX) + return TALER_MHD_reply_request_too_large (connection); + } + } } GNUNET_async_scope_enter (&rc->async_scope_id, diff --git a/src/exchange/taler-exchange-httpd_keys.c b/src/exchange/taler-exchange-httpd_keys.c index fbca5d652..29d964c6b 100644 --- a/src/exchange/taler-exchange-httpd_keys.c +++ b/src/exchange/taler-exchange-httpd_keys.c @@ -1557,7 +1557,7 @@ get_date_string (struct GNUNET_TIME_Absolute at, * @return #GNUNET_OK on success */ static enum GNUNET_GenericReturnValue -setup_general_response_headers (const struct TEH_KeyStateHandle *ksh, +setup_general_response_headers (struct TEH_KeyStateHandle *ksh, struct MHD_Response *response) { char dat[128]; @@ -1590,6 +1590,9 @@ setup_general_response_headers (const struct TEH_KeyStateHandle *ksh, MHD_add_response_header (response, MHD_HTTP_HEADER_EXPIRES, dat)); + ksh->signature_expires + = GNUNET_TIME_absolute_min (m, + ksh->signature_expires); } return GNUNET_OK; } diff --git a/src/exchange/taler-exchange-httpd_management_post_keys.c b/src/exchange/taler-exchange-httpd_management_post_keys.c index ad4cd3c04..f0c3f1f39 100644 --- a/src/exchange/taler-exchange-httpd_management_post_keys.c +++ b/src/exchange/taler-exchange-httpd_management_post_keys.c @@ -367,6 +367,8 @@ TEH_handler_management_post_keys ( TALER_EC_GENERIC_PARAMETER_MALFORMED, "array expected for denom_sigs and signkey_sigs"); } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Received /management/keys\n"); akc.nd_sigs = json_array_size (denom_sigs); akc.d_sigs = GNUNET_new_array (akc.nd_sigs, struct DenomSig); @@ -404,6 +406,8 @@ TEH_handler_management_post_keys ( { GNUNET_free (akc.d_sigs); GNUNET_JSON_parse_free (spec); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failure to handle /management/keys\n"); return ret; } akc.ns_sigs = json_array_size (signkey_sigs); @@ -440,6 +444,8 @@ TEH_handler_management_post_keys ( } if (! ok) { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failure to handle /management/keys\n"); GNUNET_free (akc.d_sigs); GNUNET_free (akc.s_sigs); GNUNET_JSON_parse_free (spec); diff --git a/src/include/taler_mhd_lib.h b/src/include/taler_mhd_lib.h index ba5a072c4..7f38ffcf5 100644 --- a/src/include/taler_mhd_lib.h +++ b/src/include/taler_mhd_lib.h @@ -30,6 +30,12 @@ #include +/** + * Maximum POST request size. + */ +#define TALER_MHD_REQUEST_BUFFER_MAX (1024 * 1024 * 16) + + /** * Global options for response generation. */ diff --git a/src/lib/exchange_api_management_post_keys.c b/src/lib/exchange_api_management_post_keys.c index e956cfd55..6b040bdaa 100644 --- a/src/lib/exchange_api_management_post_keys.c +++ b/src/lib/exchange_api_management_post_keys.c @@ -99,6 +99,10 @@ handle_post_keys_finished (void *cls, hr.ec = TALER_JSON_get_error_code (json); hr.hint = TALER_JSON_get_error_hint (json); break; + case MHD_HTTP_REQUEST_ENTITY_TOO_LARGE: + hr.ec = TALER_JSON_get_error_code (json); + hr.hint = TALER_JSON_get_error_hint (json); + break; default: /* unexpected response code */ GNUNET_break_op (0); diff --git a/src/mhd/mhd_parsing.c b/src/mhd/mhd_parsing.c index b55a3db32..4415c82a8 100644 --- a/src/mhd/mhd_parsing.c +++ b/src/mhd/mhd_parsing.c @@ -27,12 +27,6 @@ #include "taler_mhd_lib.h" -/** - * Maximum POST request size. - */ -#define REQUEST_BUFFER_MAX (1024 * 1024) - - /** * Process a POST request containing a JSON object. This function * realizes an MHD POST processor that will (incrementally) process @@ -65,7 +59,7 @@ TALER_MHD_parse_post_json (struct MHD_Connection *connection, { enum GNUNET_JSON_PostResult pr; - pr = GNUNET_JSON_post_parser (REQUEST_BUFFER_MAX, + pr = GNUNET_JSON_post_parser (TALER_MHD_REQUEST_BUFFER_MAX, connection, con_cls, upload_data, @@ -87,9 +81,9 @@ TALER_MHD_parse_post_json (struct MHD_Connection *connection, return GNUNET_YES; case GNUNET_JSON_PR_REQUEST_TOO_LARGE: GNUNET_break (NULL == *json); - return (MHD_NO == - TALER_MHD_reply_request_too_large - (connection)) ? GNUNET_SYSERR : GNUNET_NO; + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Closing connection, upload too large\n"); + return MHD_NO; case GNUNET_JSON_PR_JSON_INVALID: GNUNET_break (NULL == *json); return (MHD_YES == diff --git a/src/mhd/mhd_responses.c b/src/mhd/mhd_responses.c index 5b99dd128..2918440a2 100644 --- a/src/mhd/mhd_responses.c +++ b/src/mhd/mhd_responses.c @@ -419,24 +419,10 @@ TALER_MHD_reply_with_ec (struct MHD_Connection *connection, MHD_RESULT TALER_MHD_reply_request_too_large (struct MHD_Connection *connection) { - struct MHD_Response *response; - - response = MHD_create_response_from_buffer (0, - NULL, - MHD_RESPMEM_PERSISTENT); - if (NULL == response) - return MHD_NO; - TALER_MHD_add_global_headers (response); - - { - MHD_RESULT ret; - - ret = MHD_queue_response (connection, - MHD_HTTP_REQUEST_ENTITY_TOO_LARGE, - response); - MHD_destroy_response (response); - return ret; - } + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_REQUEST_ENTITY_TOO_LARGE, + TALER_EC_GENERIC_UPLOAD_EXCEEDS_LIMIT, + NULL); } From 21951eacc23611464ada18c99dfb4633b9fcc1b0 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 18:22:07 +0100 Subject: [PATCH 20/37] quick hack to see about improving melt serialization failures --- src/exchange/taler-exchange-httpd_melt.c | 82 ++++++++++++--------- src/exchangedb/plugin_exchangedb_postgres.c | 3 +- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_melt.c b/src/exchange/taler-exchange-httpd_melt.c index 3347790b5..c00eb8afe 100644 --- a/src/exchange/taler-exchange-httpd_melt.c +++ b/src/exchange/taler-exchange-httpd_melt.c @@ -183,8 +183,11 @@ refresh_check_melt (struct MHD_Connection *connection, struct TALER_Amount spent; enum GNUNET_DB_QueryStatus qs; - /* Start with cost of this melt transaction */ - spent = rmc->refresh_session.amount_with_fee; + /* Start with zero cost, as we already added this melt transaction + to the DB, so we will see it again during the queries below. */ + GNUNET_assert (GNUNET_OK == + TALER_amount_set_zero (TEH_currency, + &spent)); /* get historic transaction costs of this coin, including recoups as we might be a zombie coin */ @@ -311,43 +314,11 @@ melt_transaction (void *cls, enum GNUNET_DB_QueryStatus qs; uint32_t noreveal_index; - /* Check if we already created a matching refresh_session */ - qs = TEH_plugin->get_melt_index (TEH_plugin->cls, - &rmc->refresh_session.rc, - &noreveal_index); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) - { - TALER_LOG_DEBUG ("Coin was previously melted, returning old reply\n"); - *mhd_ret = reply_melt_success (connection, - &rmc->refresh_session.rc, - noreveal_index); - /* Note: we return "hard error" to ensure the wrapper - does not retry the transaction, and to also not generate - a "fresh" response (as we would on "success") */ - return GNUNET_DB_STATUS_HARD_ERROR; - } - if (0 > qs) - { - if (GNUNET_DB_STATUS_HARD_ERROR == qs) - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_FETCH_FAILED, - "melt index"); - return qs; - } - - /* check coin has enough funds remaining on it to cover melt cost */ - qs = refresh_check_melt (connection, - rmc, - mhd_ret); - if (0 > qs) - return qs; /* if we failed, tell caller */ - /* pick challenge and persist it */ rmc->refresh_session.noreveal_index = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_STRONG, TALER_CNC_KAPPA); - if (0 >= + if (0 > (qs = TEH_plugin->insert_melt (TEH_plugin->cls, &rmc->refresh_session))) { @@ -361,6 +332,47 @@ melt_transaction (void *cls, } return qs; } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + /* Check if we already created a matching refresh_session */ + qs = TEH_plugin->get_melt_index (TEH_plugin->cls, + &rmc->refresh_session.rc, + &noreveal_index); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) + { + TALER_LOG_DEBUG ("Coin was previously melted, returning old reply\n"); + *mhd_ret = reply_melt_success (connection, + &rmc->refresh_session.rc, + noreveal_index); + /* Note: we return "hard error" to ensure the wrapper + does not retry the transaction, and to also not generate + a "fresh" response (as we would on "success") */ + return GNUNET_DB_STATUS_HARD_ERROR; + } + if (0 > qs) + { + if (GNUNET_DB_STATUS_HARD_ERROR == qs) + *mhd_ret = TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "melt index"); + return qs; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + /* Conflict on insert, but record does not exist? + That makes no sense. */ + GNUNET_break (0); + return GNUNET_DB_STATUS_HARD_ERROR; + } + } + + /* check coin has enough funds remaining on it to cover melt cost */ + qs = refresh_check_melt (connection, + rmc, + mhd_ret); + if (0 > qs) + return qs; /* if we failed, tell caller */ return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; } diff --git a/src/exchangedb/plugin_exchangedb_postgres.c b/src/exchangedb/plugin_exchangedb_postgres.c index 4f0bc243d..33383df59 100644 --- a/src/exchangedb/plugin_exchangedb_postgres.c +++ b/src/exchangedb/plugin_exchangedb_postgres.c @@ -798,7 +798,8 @@ prepare_statements (struct PostgresClosure *pg) ",noreveal_index " ") SELECT $1, known_coin_id, $3, $4, $5, $6" " FROM known_coins" - " WHERE coin_pub=$2", + " WHERE coin_pub=$2" + " ON CONFLICT DO NOTHING", 6), /* Used in #postgres_get_melt() to fetch high-level information about a melt operation */ From e0700ad9164867c9209beec09b8001f1741eea15 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 20:33:14 +0100 Subject: [PATCH 21/37] optimize /deposit logic to minimize serialization failures (presumably) --- contrib/gana | 2 +- src/exchange/taler-exchange-httpd_db.c | 167 +++++++++++++++- src/exchange/taler-exchange-httpd_db.h | 26 +++ src/exchange/taler-exchange-httpd_deposit.c | 154 ++++++--------- src/exchange/taler-exchange-httpd_melt.c | 186 +----------------- .../taler-exchange-httpd_refreshes_reveal.c | 65 +++--- src/exchangedb/plugin_exchangedb_postgres.c | 3 +- src/lib/exchange_api_deposit.c | 2 +- src/lib/exchange_api_melt.c | 8 +- 9 files changed, 295 insertions(+), 318 deletions(-) diff --git a/contrib/gana b/contrib/gana index b7320181c..b12314034 160000 --- a/contrib/gana +++ b/contrib/gana @@ -1 +1 @@ -Subproject commit b7320181c5e0d95c6f2e2a9e5c53dce0bc1a35a8 +Subproject commit b123140349c3e3b300878d2e35cea1553c9a381d diff --git a/src/exchange/taler-exchange-httpd_db.c b/src/exchange/taler-exchange-httpd_db.c index c091f994d..383a7b297 100644 --- a/src/exchange/taler-exchange-httpd_db.c +++ b/src/exchange/taler-exchange-httpd_db.c @@ -1,6 +1,6 @@ /* This file is part of TALER - Copyright (C) 2014-2017 Taler Systems SA + Copyright (C) 2014-2017, 2021 Taler Systems SA TALER is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -28,6 +28,55 @@ #include "taler-exchange-httpd_responses.h" +/** + * Send a response for a failed request. The transaction history of the given + * coin demonstrates that the @a residual value of the coin is below the @a + * requested contribution of the coin for the operation. Thus, the exchange + * refuses the operation. + * + * @param connection the connection to send the response to + * @param coin_pub public key of the coin + * @param coin_value original value of the coin + * @param tl transaction history for the coin + * @param requested how much this coin was supposed to contribute, including fee + * @param residual remaining value of the coin (after subtracting @a tl) + * @return a MHD result code + */ +static MHD_RESULT +reply_insufficient_funds ( + struct MHD_Connection *connection, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *coin_value, + struct TALER_EXCHANGEDB_TransactionList *tl, + const struct TALER_Amount *requested, + const struct TALER_Amount *residual) +{ + json_t *history; + + history = TEH_RESPONSE_compile_transaction_history (coin_pub, + tl); + if (NULL == history) + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_EXCHANGE_GENERIC_HISTORY_DB_ERROR_INSUFFICIENT_FUNDS, + NULL); + return TALER_MHD_REPLY_JSON_PACK ( + connection, + MHD_HTTP_CONFLICT, + TALER_JSON_pack_ec (TALER_EC_EXCHANGE_GENERIC_INSUFFICIENT_FUNDS), + GNUNET_JSON_pack_data_auto ("coin_pub", + coin_pub), + TALER_JSON_pack_amount ("original_value", + coin_value), + TALER_JSON_pack_amount ("residual_value", + residual), + TALER_JSON_pack_amount ("requested_value", + requested), + GNUNET_JSON_pack_array_steal ("history", + history)); +} + + /** * How often should we retry a transaction before giving up * (for transactions resulting in serialization/dead locks only). @@ -114,6 +163,122 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin, } +enum GNUNET_DB_QueryStatus +TEH_check_coin_balance (struct MHD_Connection *connection, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *coin_value, + const struct TALER_Amount *op_cost, + bool check_recoup, + bool zombie_required, + MHD_RESULT *mhd_ret) +{ + struct TALER_EXCHANGEDB_TransactionList *tl; + struct TALER_Amount spent; + enum GNUNET_DB_QueryStatus qs; + + /* Start with zero cost, as we already added this melt transaction + to the DB, so we will see it again during the queries below. */ + GNUNET_assert (GNUNET_OK == + TALER_amount_set_zero (TEH_currency, + &spent)); + + /* get historic transaction costs of this coin, including recoups as + we might be a zombie coin */ + qs = TEH_plugin->get_coin_transactions (TEH_plugin->cls, + coin_pub, + check_recoup, + &tl); + if (0 > qs) + { + if (GNUNET_DB_STATUS_HARD_ERROR == qs) + *mhd_ret = TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "coin transaction history"); + return qs; + } + if (zombie_required) + { + /* The denomination key is only usable for a melt if this is a true + zombie coin, i.e. it was refreshed and the resulting fresh coin was + then recouped. Check that this is truly the case. */ + for (struct TALER_EXCHANGEDB_TransactionList *tp = tl; + NULL != tp; + tp = tp->next) + { + if (TALER_EXCHANGEDB_TT_OLD_COIN_RECOUP == tp->type) + { + zombie_required = false; /* clear flag: was satisfied! */ + break; + } + } + if (zombie_required) + { + /* zombie status not satisfied */ + GNUNET_break_op (0); + TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, + tl); + *mhd_ret = TALER_MHD_reply_with_error (connection, + MHD_HTTP_BAD_REQUEST, + TALER_EC_EXCHANGE_MELT_COIN_EXPIRED_NO_ZOMBIE, + NULL); + return GNUNET_DB_STATUS_HARD_ERROR; + } + } + if (GNUNET_OK != + TALER_EXCHANGEDB_calculate_transaction_list_totals (tl, + &spent, + &spent)) + { + GNUNET_break (0); + TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, + tl); + *mhd_ret = TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_EXCHANGE_GENERIC_COIN_HISTORY_COMPUTATION_FAILED, + NULL); + return GNUNET_DB_STATUS_HARD_ERROR; + } + + /* Refuse to refresh when the coin's value is insufficient + for the cost of all transactions. */ + if (0 > TALER_amount_cmp (coin_value, + &spent)) + { + struct TALER_Amount coin_residual; + struct TALER_Amount spent_already; + + /* First subtract the melt cost from 'spent' to + compute the total amount already spent of the coin */ + GNUNET_assert (0 <= + TALER_amount_subtract (&spent_already, + &spent, + op_cost)); + /* The residual coin value is the original coin value minus + what we have spent (before the melt) */ + GNUNET_assert (0 <= + TALER_amount_subtract (&coin_residual, + coin_value, + &spent_already)); + *mhd_ret = reply_insufficient_funds ( + connection, + coin_pub, + coin_value, + tl, + op_cost, + &coin_residual); + TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, + tl); + return GNUNET_DB_STATUS_HARD_ERROR; + } + + /* we're good, coin has sufficient funds to be melted */ + TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, + tl); + return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; +} + + enum GNUNET_GenericReturnValue TEH_DB_run_transaction (struct MHD_Connection *connection, const char *name, diff --git a/src/exchange/taler-exchange-httpd_db.h b/src/exchange/taler-exchange-httpd_db.h index 83302235d..60885dbd1 100644 --- a/src/exchange/taler-exchange-httpd_db.h +++ b/src/exchange/taler-exchange-httpd_db.h @@ -41,6 +41,32 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin, MHD_RESULT *mhd_ret); +/** + * Check that a coin has an adequate balance so that we can + * commit the current transaction. If the balance is + * insufficient for all transactions associated with the + * coin, return a hard error. + * + * @param connection HTTP connection to report hard errors on + * @param coin_pub coin to analyze + * @param coin_value total value of the original coin (by denomination) + * @param op_cost cost of the current operation (for error reporting) + * @param check_recoup should we include recoup transactions in the check + * @param zombie_required additional requirement that the coin must + * be a zombie coin, or also hard failure + * @param[out] mhd_ret set to response status code, on hard error only + * @return transaction status + */ +enum GNUNET_DB_QueryStatus +TEH_check_coin_balance (struct MHD_Connection *connection, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *coin_value, + const struct TALER_Amount *op_cost, + bool check_recoup, + bool zombie_required, + MHD_RESULT *mhd_ret); + + /** * Function implementing a database transaction. Runs the transaction * logic; IF it returns a non-error code, the transaction logic MUST diff --git a/src/exchange/taler-exchange-httpd_deposit.c b/src/exchange/taler-exchange-httpd_deposit.c index d549a1fa6..43acad44b 100644 --- a/src/exchange/taler-exchange-httpd_deposit.c +++ b/src/exchange/taler-exchange-httpd_deposit.c @@ -162,113 +162,85 @@ deposit_transaction (void *cls, enum GNUNET_DB_QueryStatus qs; struct TALER_Amount deposit_fee; - /* Check for idempotency: did we get this request before? */ - qs = TEH_plugin->have_deposit (TEH_plugin->cls, - deposit, - &deposit_fee, - &dc->exchange_timestamp); + /* begin optimistically: assume this is a new deposit */ + qs = TEH_plugin->insert_deposit (TEH_plugin->cls, + dc->exchange_timestamp, + deposit); if (qs < 0) { - if (GNUNET_DB_STATUS_HARD_ERROR == qs) + if (GNUNET_DB_STATUS_SOFT_ERROR == qs) + return qs; + TALER_LOG_WARNING ("Failed to store /deposit information in database\n"); + *mhd_ret = TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_STORE_FAILED, + NULL); + return qs; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + /* Check for idempotency: did we get this request before? */ + qs = TEH_plugin->have_deposit (TEH_plugin->cls, + deposit, + &deposit_fee, + &dc->exchange_timestamp); + if (qs < 0) { + if (GNUNET_DB_STATUS_SOFT_ERROR == qs) + return qs; *mhd_ret = TALER_MHD_reply_with_error (connection, MHD_HTTP_INTERNAL_SERVER_ERROR, TALER_EC_GENERIC_DB_FETCH_FAILED, "have_deposit"); return GNUNET_DB_STATUS_HARD_ERROR; } - return qs; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) - { - struct TALER_Amount amount_without_fee; - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "/deposit replay, accepting again!\n"); - GNUNET_assert (0 <= - TALER_amount_subtract (&amount_without_fee, - &deposit->amount_with_fee, - &deposit_fee)); - *mhd_ret = reply_deposit_success (connection, - &deposit->coin.coin_pub, - &dc->h_wire, - NULL /* h_extensions! */, - &deposit->h_contract_terms, - dc->exchange_timestamp, - deposit->refund_deadline, - deposit->wire_deadline, - &deposit->merchant_pub, - &amount_without_fee); - return GNUNET_DB_STATUS_HARD_ERROR; - } - - /* Start with fee for THIS transaction */ - spent = deposit->amount_with_fee; - /* add cost of all previous transactions; skip RECOUP as revoked - denominations are not eligible for deposit, and if we are the old coin - pub of a revoked coin (aka a zombie), then ONLY refresh is allowed. */ - { - struct TALER_EXCHANGEDB_TransactionList *tl; - - qs = TEH_plugin->get_coin_transactions (TEH_plugin->cls, - &deposit->coin.coin_pub, - GNUNET_NO, - &tl); - if (0 > qs) + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) { - if (GNUNET_DB_STATUS_HARD_ERROR == qs) - *mhd_ret = TALER_MHD_reply_with_error ( - connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_FETCH_FAILED, - NULL); - return qs; - } - if (GNUNET_OK != - TALER_EXCHANGEDB_calculate_transaction_list_totals (tl, - &spent, /* starting offset */ - &spent /* result */)) - { - TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, - tl); - *mhd_ret = TALER_MHD_reply_with_error ( - connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_INVARIANT_FAILURE, - NULL); + /* Conflict on insert, but record does not exist? + That makes no sense. */ + GNUNET_break (0); return GNUNET_DB_STATUS_HARD_ERROR; } - /* Check that cost of all transactions (including the current one) is - smaller (or equal) than the value of the coin. */ - if (0 < TALER_amount_cmp (&spent, - &dc->value)) + { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Deposited coin has insufficient funds left!\n"); - *mhd_ret = TEH_RESPONSE_reply_coin_insufficient_funds (connection, - TALER_EC_EXCHANGE_DEPOSIT_INSUFFICIENT_FUNDS, - &deposit->coin. - coin_pub, - tl); - TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, - tl); + struct TALER_Amount amount_without_fee; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "/deposit replay, accepting again!\n"); + GNUNET_assert (0 <= + TALER_amount_subtract (&amount_without_fee, + &deposit->amount_with_fee, + &deposit_fee)); + *mhd_ret = reply_deposit_success (connection, + &deposit->coin.coin_pub, + &dc->h_wire, + NULL /* h_extensions! */, + &deposit->h_contract_terms, + dc->exchange_timestamp, + deposit->refund_deadline, + deposit->wire_deadline, + &deposit->merchant_pub, + &amount_without_fee); + /* Note: we return "hard error" to ensure the wrapper + does not retry the transaction, and to also not generate + a "fresh" response (as we would on "success") */ return GNUNET_DB_STATUS_HARD_ERROR; } - TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, - tl); } - qs = TEH_plugin->insert_deposit (TEH_plugin->cls, - dc->exchange_timestamp, - deposit); - if (GNUNET_DB_STATUS_HARD_ERROR == qs) - { - TALER_LOG_WARNING ("Failed to store /deposit information in database\n"); - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_STORE_FAILED, - NULL); - } - return qs; + + /* Start with zero cost, as we already added this melt transaction + to the DB, so we will see it again during the queries below. */ + GNUNET_assert (GNUNET_OK == + TALER_amount_set_zero (TEH_currency, + &spent)); + + return TEH_check_coin_balance (connection, + &deposit->coin.coin_pub, + &dc->value, + &deposit->amount_with_fee, + false, /* no need for recoup */ + false, /* no need for zombie */ + mhd_ret); } diff --git a/src/exchange/taler-exchange-httpd_melt.c b/src/exchange/taler-exchange-httpd_melt.c index c00eb8afe..ab7bed295 100644 --- a/src/exchange/taler-exchange-httpd_melt.c +++ b/src/exchange/taler-exchange-httpd_melt.c @@ -33,56 +33,6 @@ #include "taler_exchangedb_lib.h" -/** - * Send a response for a failed "melt" request. The - * transaction history of the given coin demonstrates that the - * @a residual value of the coin is below the @a requested - * contribution of the coin for the melt. Thus, the exchange - * refuses the melt operation. - * - * @param connection the connection to send the response to - * @param coin_pub public key of the coin - * @param coin_value original value of the coin - * @param tl transaction history for the coin - * @param requested how much this coin was supposed to contribute, including fee - * @param residual remaining value of the coin (after subtracting @a tl) - * @return a MHD result code - */ -static MHD_RESULT -reply_melt_insufficient_funds ( - struct MHD_Connection *connection, - const struct TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_Amount *coin_value, - struct TALER_EXCHANGEDB_TransactionList *tl, - const struct TALER_Amount *requested, - const struct TALER_Amount *residual) -{ - json_t *history; - - history = TEH_RESPONSE_compile_transaction_history (coin_pub, - tl); - if (NULL == history) - return TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_EXCHANGE_MELT_HISTORY_DB_ERROR_INSUFFICIENT_FUNDS, - NULL); - return TALER_MHD_REPLY_JSON_PACK ( - connection, - MHD_HTTP_CONFLICT, - TALER_JSON_pack_ec (TALER_EC_EXCHANGE_MELT_INSUFFICIENT_FUNDS), - GNUNET_JSON_pack_data_auto ("coin_pub", - coin_pub), - TALER_JSON_pack_amount ("original_value", - coin_value), - TALER_JSON_pack_amount ("residual_value", - residual), - TALER_JSON_pack_amount ("requested_value", - requested), - GNUNET_JSON_pack_array_steal ("history", - history)); -} - - /** * Send a response to a "melt" request. * @@ -165,127 +115,6 @@ struct MeltContext }; -/** - * Check that the coin has sufficient funds left for the selected - * melt operation. - * - * @param connection the connection to send errors to - * @param[in,out] rmc melt context - * @param[out] mhd_ret status code to return to MHD on hard error - * @return transaction status code - */ -static enum GNUNET_DB_QueryStatus -refresh_check_melt (struct MHD_Connection *connection, - struct MeltContext *rmc, - MHD_RESULT *mhd_ret) -{ - struct TALER_EXCHANGEDB_TransactionList *tl; - struct TALER_Amount spent; - enum GNUNET_DB_QueryStatus qs; - - /* Start with zero cost, as we already added this melt transaction - to the DB, so we will see it again during the queries below. */ - GNUNET_assert (GNUNET_OK == - TALER_amount_set_zero (TEH_currency, - &spent)); - - /* get historic transaction costs of this coin, including recoups as - we might be a zombie coin */ - qs = TEH_plugin->get_coin_transactions (TEH_plugin->cls, - &rmc->refresh_session.coin.coin_pub, - GNUNET_YES, - &tl); - if (0 > qs) - { - if (GNUNET_DB_STATUS_HARD_ERROR == qs) - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_FETCH_FAILED, - "coin transaction history"); - return qs; - } - if (rmc->zombie_required) - { - /* The denomination key is only usable for a melt if this is a true - zombie coin, i.e. it was refreshed and the resulting fresh coin was - then recouped. Check that this is truly the case. */ - for (struct TALER_EXCHANGEDB_TransactionList *tp = tl; - NULL != tp; - tp = tp->next) - { - if (TALER_EXCHANGEDB_TT_OLD_COIN_RECOUP == tp->type) - { - rmc->zombie_required = false; /* clear flag: was satisfied! */ - break; - } - } - if (rmc->zombie_required) - { - /* zombie status not satisfied */ - GNUNET_break_op (0); - TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, - tl); - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_BAD_REQUEST, - TALER_EC_EXCHANGE_MELT_COIN_EXPIRED_NO_ZOMBIE, - NULL); - return GNUNET_DB_STATUS_HARD_ERROR; - } - } - if (GNUNET_OK != - TALER_EXCHANGEDB_calculate_transaction_list_totals (tl, - &spent, - &spent)) - { - GNUNET_break (0); - TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, - tl); - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_EXCHANGE_MELT_COIN_HISTORY_COMPUTATION_FAILED, - NULL); - return GNUNET_DB_STATUS_HARD_ERROR; - } - - /* Refuse to refresh when the coin's value is insufficient - for the cost of all transactions. */ - if (0 > TALER_amount_cmp (&rmc->coin_value, - &spent)) - { - struct TALER_Amount coin_residual; - struct TALER_Amount spent_already; - - /* First subtract the melt cost from 'spent' to - compute the total amount already spent of the coin */ - GNUNET_assert (0 <= - TALER_amount_subtract (&spent_already, - &spent, - &rmc->refresh_session.amount_with_fee)); - /* The residual coin value is the original coin value minus - what we have spent (before the melt) */ - GNUNET_assert (0 <= - TALER_amount_subtract (&coin_residual, - &rmc->coin_value, - &spent_already)); - *mhd_ret = reply_melt_insufficient_funds ( - connection, - &rmc->refresh_session.coin.coin_pub, - &rmc->coin_value, - tl, - &rmc->refresh_session.amount_with_fee, - &coin_residual); - TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, - tl); - return GNUNET_DB_STATUS_HARD_ERROR; - } - - /* we're good, coin has sufficient funds to be melted */ - TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, - tl); - return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; -} - - /** * Execute a "melt". We have been given a list of valid * coins and a request to melt them into the given @a @@ -366,14 +195,13 @@ melt_transaction (void *cls, return GNUNET_DB_STATUS_HARD_ERROR; } } - - /* check coin has enough funds remaining on it to cover melt cost */ - qs = refresh_check_melt (connection, - rmc, - mhd_ret); - if (0 > qs) - return qs; /* if we failed, tell caller */ - return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + return TEH_check_coin_balance (connection, + &rmc->refresh_session.coin.coin_pub, + &rmc->coin_value, + &rmc->refresh_session.amount_with_fee, + true, + rmc->zombie_required, + mhd_ret); } diff --git a/src/exchange/taler-exchange-httpd_refreshes_reveal.c b/src/exchange/taler-exchange-httpd_refreshes_reveal.c index 08587a516..6c1766feb 100644 --- a/src/exchange/taler-exchange-httpd_refreshes_reveal.c +++ b/src/exchange/taler-exchange-httpd_refreshes_reveal.c @@ -105,6 +105,11 @@ struct RevealContext */ struct TALER_TransferPrivateKeyP transfer_privs[TALER_CNC_KAPPA - 1]; + /** + * Melt data for our session we got from the database for @e rc. + */ + struct TALER_EXCHANGEDB_Melt melt; + /** * Denominations being requested. */ @@ -266,35 +271,6 @@ refreshes_reveal_transaction (void *cls, MHD_RESULT *mhd_ret) { struct RevealContext *rctx = cls; - struct TALER_EXCHANGEDB_Melt melt; - enum GNUNET_DB_QueryStatus qs; - - /* Obtain basic information about the refresh operation and what - gamma we committed to. */ - // FIXME: why do we do 'get_melt' twice? - qs = TEH_plugin->get_melt (TEH_plugin->cls, - &rctx->rc, - &melt); - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_NOT_FOUND, - TALER_EC_EXCHANGE_REFRESHES_REVEAL_SESSION_UNKNOWN, - NULL); - return GNUNET_DB_STATUS_HARD_ERROR; - } - if (GNUNET_DB_STATUS_SOFT_ERROR == qs) - return qs; - if ( (GNUNET_DB_STATUS_HARD_ERROR == qs) || - (melt.session.noreveal_index >= TALER_CNC_KAPPA) ) - { - GNUNET_break (0); - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_FETCH_FAILED, - "melt"); - return GNUNET_DB_STATUS_HARD_ERROR; - } /* Verify commitment */ { @@ -310,7 +286,7 @@ refreshes_reveal_transaction (void *cls, { struct TALER_RefreshCommitmentEntry *rce = &rcs[i]; - if (i == melt.session.noreveal_index) + if (i == rctx->melt.session.noreveal_index) { /* Take these coin envelopes from the client */ rce->transfer_pub = rctx->gamma_tp; @@ -327,7 +303,7 @@ refreshes_reveal_transaction (void *cls, GNUNET_CRYPTO_ecdhe_key_get_public (&tpriv->ecdhe_priv, &rce->transfer_pub.ecdhe_pub); TALER_link_reveal_transfer_secret (tpriv, - &melt.session.coin.coin_pub, + &rctx->melt.session.coin.coin_pub, &ts); rce->new_coins = GNUNET_new_array (rctx->num_fresh_coins, struct TALER_RefreshCoinData); @@ -356,15 +332,15 @@ refreshes_reveal_transaction (void *cls, TALER_CNC_KAPPA, rctx->num_fresh_coins, rcs, - &melt.session.coin.coin_pub, - &melt.session.amount_with_fee); + &rctx->melt.session.coin.coin_pub, + &rctx->melt.session.amount_with_fee); /* Free resources allocated above */ for (unsigned int i = 0; imelt.session.noreveal_index) continue; /* This offset is special: not allocated! */ for (unsigned int j = 0; jnum_fresh_coins; j++) { @@ -395,7 +371,7 @@ refreshes_reveal_transaction (void *cls, { struct TALER_Amount refresh_cost; - refresh_cost = melt.melt_fee; + refresh_cost = rctx->melt.melt_fee; for (unsigned int i = 0; inum_fresh_coins; i++) { struct TALER_Amount total; @@ -418,7 +394,7 @@ refreshes_reveal_transaction (void *cls, } } if (0 < TALER_amount_cmp (&refresh_cost, - &melt.session.amount_with_fee)) + &rctx->melt.session.amount_with_fee)) { GNUNET_break_op (0); *mhd_ret = TALER_MHD_reply_with_error (connection, @@ -505,7 +481,6 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection, struct TALER_DenominationHash dk_h[num_fresh_coins]; struct TALER_RefreshCoinData rcds[num_fresh_coins]; struct TALER_CoinSpendSignatureP link_sigs[num_fresh_coins]; - struct TALER_EXCHANGEDB_Melt melt; enum GNUNET_GenericReturnValue res; MHD_RESULT ret; struct TEH_KeyStateHandle *ksh; @@ -612,11 +587,10 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection, { enum GNUNET_DB_QueryStatus qs; - // FIXME: why do we do 'get_melt' twice? if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != (qs = TEH_plugin->get_melt (TEH_plugin->cls, &rctx->rc, - &melt))) + &rctx->melt))) { switch (qs) { @@ -643,6 +617,17 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection, } goto cleanup; } + /* Obtain basic information about the refresh operation and what + gamma we committed to. */ + if (rctx->melt.session.noreveal_index >= TALER_CNC_KAPPA) + { + GNUNET_break (0); + ret = TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "melt"); + goto cleanup; + } } /* Parse link signatures array */ for (unsigned int i = 0; igamma_tp, rcds[i].coin_ev, rcds[i].coin_ev_size, - &melt.session.coin.coin_pub, + &rctx->melt.session.coin.coin_pub, &link_sigs[i])) { GNUNET_break_op (0); diff --git a/src/exchangedb/plugin_exchangedb_postgres.c b/src/exchangedb/plugin_exchangedb_postgres.c index 33383df59..73a03e751 100644 --- a/src/exchangedb/plugin_exchangedb_postgres.c +++ b/src/exchangedb/plugin_exchangedb_postgres.c @@ -1034,7 +1034,8 @@ prepare_statements (struct PostgresClosure *pg) ") SELECT known_coin_id, $2, $3, $4, $5, $6, " " $7, $8, $9, $10, $11, $12, $13" " FROM known_coins" - " WHERE coin_pub=$1;", + " WHERE coin_pub=$1" + " ON CONFLICT DO NOTHING;", 13), /* Fetch an existing deposit request, used to ensure idempotency during /deposit processing. Used in #postgres_have_deposit(). */ diff --git a/src/lib/exchange_api_deposit.c b/src/lib/exchange_api_deposit.c index 8b93f56ef..3fd3353b2 100644 --- a/src/lib/exchange_api_deposit.c +++ b/src/lib/exchange_api_deposit.c @@ -263,7 +263,7 @@ verify_deposit_signature_conflict ( ec = TALER_JSON_get_error_code (json); switch (ec) { - case TALER_EC_EXCHANGE_DEPOSIT_INSUFFICIENT_FUNDS: + case TALER_EC_EXCHANGE_GENERIC_INSUFFICIENT_FUNDS: if (0 > TALER_amount_add (&total, &total, diff --git a/src/lib/exchange_api_melt.c b/src/lib/exchange_api_melt.c index f3032e8b2..6ef9f4cbc 100644 --- a/src/lib/exchange_api_melt.c +++ b/src/lib/exchange_api_melt.c @@ -97,7 +97,7 @@ struct TALER_EXCHANGE_MeltHandle * @param[out] noreveal_index set to the noreveal index selected by the exchange * @return #GNUNET_OK if the signature is valid, #GNUNET_SYSERR if not */ -static int +static enum GNUNET_GenericReturnValue verify_melt_signature_ok (struct TALER_EXCHANGE_MeltHandle *mh, const json_t *json, struct TALER_ExchangePublicKeyP *exchange_pub, @@ -208,7 +208,7 @@ verify_melt_signature_denom_conflict (struct TALER_EXCHANGE_MeltHandle *mh, * @param json json reply with the signature(s) and transaction history * @return #GNUNET_OK if the signature(s) is valid, #GNUNET_SYSERR if not */ -static int +static enum GNUNET_GenericReturnValue verify_melt_signature_spend_conflict (struct TALER_EXCHANGE_MeltHandle *mh, const json_t *json) { @@ -282,7 +282,7 @@ verify_melt_signature_spend_conflict (struct TALER_EXCHANGE_MeltHandle *mh, ec = TALER_JSON_get_error_code (json); switch (ec) { - case TALER_EC_EXCHANGE_MELT_INSUFFICIENT_FUNDS: + case TALER_EC_EXCHANGE_GENERIC_INSUFFICIENT_FUNDS: /* check if melt operation was really too expensive given history */ if (0 > TALER_amount_add (&total, @@ -379,7 +379,7 @@ handle_melt_finished (void *cls, hr.ec = TALER_JSON_get_error_code (j); switch (hr.ec) { - case TALER_EC_EXCHANGE_MELT_INSUFFICIENT_FUNDS: + case TALER_EC_EXCHANGE_GENERIC_INSUFFICIENT_FUNDS: /* Double spending; check signatures on transaction history */ if (GNUNET_OK != verify_melt_signature_spend_conflict (mh, From 7fdcec4c3c401bf18e24234f4323aa9545c0eb72 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 20:52:23 +0100 Subject: [PATCH 22/37] -complete 'melt.sql', in theory --- src/exchangedb/melt.sql | 243 +++++++++++----------------------------- 1 file changed, 67 insertions(+), 176 deletions(-) diff --git a/src/exchangedb/melt.sql b/src/exchangedb/melt.sql index af1aa8d4a..fc6d24d45 100644 --- a/src/exchangedb/melt.sql +++ b/src/exchangedb/melt.sql @@ -5,20 +5,15 @@ -- Check patch versioning is in place. -- SELECT _v.register_patch('exchange-000x', NULL, NULL); -CREATE OR REPLACE FUNCTION exchange_do_melt( + +CREATE OR REPLACE FUNCTION exchange_check_coin_balance( IN denom_val INT8, -- value of the denomination of the coin IN denom_frac INT4, -- value of the denomination of the coin - IN amount_val INT8, -- requested melt amount (with fee) - IN amount_frac INT4, -- requested melt amount (with fee) - IN in_rc BYTEA, -- refresh session hash IN in_coin_pub BYTEA, -- coin public key - IN coin_sig BYTEA, -- melt signature - IN in_noreveal_index INT4, -- suggested random noreveal index + IN check_recoup BOOLEAN, -- do we need to check the recoup table? IN zombie_required BOOLEAN, -- do we need a zombie coin? - OUT out_noreval_index INT4, -- noreveal index to actually use OUT balance_ok BOOLEAN, -- balance satisfied? - OUT zombie_ok BOOLEAN, -- zombie satisfied? - OUT melt_ok BOOLEAN) -- everything OK? + OUT zombie_ok BOOLEAN) -- zombie satisfied? LANGUAGE plpgsql AS $$ DECLARE @@ -37,6 +32,9 @@ DECLARE unspent_frac INT8; -- how much of coin was refunded? BEGIN +-- Note: possible future optimization: get the coin_uuid from the previous +-- 'ensure_coin_known' and pass that here instead of the coin_pub. Might help +-- a tiny bit with performance. SELECT known_coin_id INTO coin_uuid FROM known_coins WHERE coin_pub=in_coin_pub; @@ -44,49 +42,17 @@ SELECT known_coin_id INTO coin_uuid IF NOT FOUND THEN -- coin unknown, should be impossible! - out_noreveal_index=-1; balance_ok=FALSE; zombie_ok=FALSE; - melt_ok=FALSE; ASSERT false, 'coin unknown'; RETURN; END IF; --- We optimistically insert, and then on conflict declare --- the query successful due to idempotency. -INSERT INTO refresh_commitments - (rc - ,old_known_coin_id - ,old_coin_sig - ,amount_with_fee_val - ,amount_with_fee_frac - ,noreveal_index) -VALUES - (in_rc - ,coin_uuid - ,coin_sig - ,amount_val - ,amount_frac - ,in_noreveal_index) -ON CONFLICT DO NOTHING; -IF FOUND -THEN - -- already melted, get noreveal_index - SELECT noreveal_index INTO out_noreveal_index - FROM refresh_commitments - WHERE rc=in_rc ; - balance_ok=TRUE; - zombie_ok=TRUE; - melt_ok=TRUE; - RETURN; -END IF; - --- Need to check for sufficient balance... spent_val = 0; spent_frac = 0; -unspent_val = 0; -unspent_frac = 0; +unspent_val = denom_val; +unspent_frac = denom_frac; SELECT SUM(amount_with_fee_val) -- overflow here is not plausible @@ -124,156 +90,81 @@ SELECT unspent_val = unspent_val + tmp_val; unspent_frac = unspent_frac + tmp_frac; -SELECT - SUM(amount_val) -- overflow here is not plausible - ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM recoup_refresh - WHERE known_coin_id=coin_uuid; - -unspent_val = unspent_val + tmp_val; -unspent_frac = unspent_frac + tmp_frac; - -SELECT - SUM(amount_val) -- overflow here is not plausible - ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM recoup - WHERE known_coin_id=coin_uuid; - -spent_val = spent_val + tmp_val; -spent_frac = spent_frac + tmp_frac; - -SELECT - SUM(amount_val) -- overflow here is not plausible - ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM recoup_refresh - JOIN refresh_revealed_coins rrc - USING (rrc_serial) - JOIN refresh_commitments rfc - ON (rrc.melt_serial_id = rfc.melt_serial_id) - WHERE rfc.old_known_coin_id=coin_uuid; - -spent_val = spent_val + tmp_val; -spent_frac = spent_frac + tmp_frac; - - -------------------- TBD from here - -SELECT - reserve_uuid - ,current_balance_val - ,current_balance_frac_uuid - ,expiration_date - ,gc_date - INTO - reserve_uuid - ,reserve_val - ,reserve_frac - ,reserve_gc - FROM reserves - WHERE reserve_pub=reserve_pub; - -IF NOT FOUND +-- Note: even if 'check_recoup' is true, the tables below +-- are in practice likely empty (as they only apply if +-- the exchange (ever) had to revoke keys). +IF check_recoup THEN - -- reserve unknown - reserve_found=FALSE; - balance_ok=FALSE; - kyc_ok=FALSE; - RETURN; -END IF; --- We optimistically insert, and then on conflict declare --- the query successful due to idempotency. -INSERT INTO reserves_out - (h_blind_ev - ,denom_serial - ,denom_sig - ,reserve_uuid - ,reserve_sig - ,execution_date - ,amount_with_fee_val - ,amount_with_fee_frac) -VALUES - (h_coin_envelope - ,denom_serial - ,denom_sig - ,reserve_uuid - ,reserve_sig - ,now - ,amount_val - ,amount_frac) -ON CONFLICT DO NOTHING; + SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup_refresh + WHERE known_coin_id=coin_uuid; -IF NOT FOUND -THEN - -- idempotent query, all constraints must be satisfied - reserve_found=TRUE; - balance_ok=TRUE; - kyc_ok=TRUE; - RETURN; -END IF; + unspent_val = unspent_val + tmp_val; + unspent_frac = unspent_frac + tmp_frac; --- Check reserve balance is sufficient. -IF (reserve_val > amount_val) -THEN - IF (reserve_frac > amount_frac) + SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup + WHERE known_coin_id=coin_uuid; + + spent_val = spent_val + tmp_val; + spent_frac = spent_frac + tmp_frac; + + SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup_refresh + JOIN refresh_revealed_coins rrc + USING (rrc_serial) + JOIN refresh_commitments rfc + ON (rrc.melt_serial_id = rfc.melt_serial_id) + WHERE rfc.old_known_coin_id=coin_uuid; + + spent_val = spent_val + tmp_val; + spent_frac = spent_frac + tmp_frac; + + IF ( (0 < tmp_val) OR (0 < tmp_frac) ) THEN - reserve_val=reserve_val - amount_val; - reserve_frac=reserve_frac - amount_frac; - ELSE - reserve_val=reserve_val - amount_val - 1; - reserve_frac=reserve_frac + 100000000 - amount_frac; + -- There was a transaction that justifies the zombie + -- status, clear the flag + zombie_required=FALSE; END IF; + +END IF; + + +-- Actually check if the coin balance is sufficient. Verbosely. ;-) +IF (unspent_val > spent_val) +THEN + balance_ok=TRUE; ELSE IF (reserve_val == amount_val) AND (reserve_frac >= amount_frac) THEN - reserve_val=0; - reserve_frac=reserve_frac - amount_frac; + balance_ok=TRUE; ELSE - reserve_found=TRUE; balance_ok=FALSE; - kyc_ok=FALSE; -- we do not really know or care - RETURN; END IF; END IF; --- Calculate new expiration dates. -min_reserve_gc=MAX(min_reserve_gc,reserve_gc); - --- Update reserve balance. -UPDATE reserves SET - gc_date=min_reserve_gc - ,current_balance_val=reserve_val - ,current_balance_frac=reserve_frac -WHERE - reserve_uuid=reserve_uuid; - -reserve_found=TRUE; -balance_ok=TRUE; - --- Obtain KYC status based on the last wire transfer into --- this reserve. FIXME: likely not adequate for reserves that got P2P transfers! -SELECT kyc_ok - INTO kyc_ok - FROM reserves_in - JOIN wire_targets USING (wire_target_serial_id) - WHERE reserve_uuid=reserve_uuid - LIMIT 1; -- limit 1 should not be required (without p2p transfers) - - +zombie_ok = NOT zombie_required; END $$; -COMMENT ON FUNCTION exchange_do_melt(INT8, INT4, BYTEA, BYTEA, BYTEA, BYTEA, BYTEA, INT8, INT8) - IS 'Checks whether the coin has sufficient balance for a melt operation (or the request is repeated and was previously approved) and if so updates the database with the result'; +COMMENT ON FUNCTION exchange_check_coin_balance(INT8, INT4, BYTEA, BOOLEAN, BOOLEAN) + IS 'Checks whether the coin has sufficient balance for all the operations associated with it'; -- Complete transaction From 5433c2a475a9325d0b9f0272faa197a6f8e6d24b Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 8 Dec 2021 21:14:36 +0100 Subject: [PATCH 23/37] be more backwards-compatible --- src/testing/testing_api_cmd_oauth.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/testing/testing_api_cmd_oauth.c b/src/testing/testing_api_cmd_oauth.c index 9c2fef8e4..eaf7f9068 100644 --- a/src/testing/testing_api_cmd_oauth.c +++ b/src/testing/testing_api_cmd_oauth.c @@ -79,7 +79,7 @@ append (char **target, } -static enum MHD_Result +static MHD_RESULT handle_post (void *cls, enum MHD_ValueKind kind, const char *key, @@ -160,7 +160,7 @@ handle_post (void *cls, * #MHD_NO if the socket must be closed due to a serious * error while handling the request */ -static enum MHD_Result +static MHD_RESULT handler_cb (void *cls, struct MHD_Connection *connection, const char *url, @@ -210,7 +210,7 @@ handler_cb (void *cls, } if (0 != *upload_data_size) { - enum MHD_Result ret; + MHD_RESULT ret; ret = MHD_post_process (rc->pp, upload_data, From dcb73693dd69767828b975dc603e6b0dd2d5408d Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Thu, 9 Dec 2021 10:38:02 +0100 Subject: [PATCH 24/37] crypto worker: eddsa signing --- src/exchange-tools/taler-crypto-worker.c | 86 ++++++++++++------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/src/exchange-tools/taler-crypto-worker.c b/src/exchange-tools/taler-crypto-worker.c index 674724eda..6690b912b 100644 --- a/src/exchange-tools/taler-crypto-worker.c +++ b/src/exchange-tools/taler-crypto-worker.c @@ -137,12 +137,53 @@ run (void *cls, "sent response\n"); continue; } + if (0 == strcmp ("eddsa_sign", + op)) + { + struct GNUNET_CRYPTO_EddsaSignature sig; + struct GNUNET_CRYPTO_EccSignaturePurpose *msg; + struct GNUNET_CRYPTO_EddsaPrivateKey priv; + size_t msg_size; + json_t *resp; + struct GNUNET_JSON_Specification eddsa_sign_spec[] = { + GNUNET_JSON_spec_fixed_auto ("priv", + &priv), + GNUNET_JSON_spec_varsize ("msg", + (void **) &msg, + &msg_size), + GNUNET_JSON_spec_end () + }; + if (GNUNET_OK != GNUNET_JSON_parse (args, + eddsa_sign_spec, + NULL, + NULL)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "malformed op args\n"); + global_ret = 1; + return; + } + GNUNET_CRYPTO_eddsa_sign_ ( + &priv, + msg, + &sig + ); + resp = GNUNET_JSON_PACK ( + GNUNET_JSON_pack_data_auto ("sig", &sig) + ); + json_dumpf (resp, stdout, JSON_COMPACT); + printf ("\n"); + fflush (stdout); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "sent response\n"); + continue; + } if (0 == strcmp ("setup_refresh_planchet", op)) { struct TALER_TransferSecretP transfer_secret; uint32_t coin_index; json_t *resp; - struct GNUNET_JSON_Specification eddsa_verify_spec[] = { + struct GNUNET_JSON_Specification setup_refresh_planchet_spec[] = { GNUNET_JSON_spec_uint32 ("coin_index", &coin_index), GNUNET_JSON_spec_fixed_auto ("transfer_secret", @@ -154,7 +195,7 @@ run (void *cls, if (GNUNET_OK != GNUNET_JSON_parse (args, - eddsa_verify_spec, + setup_refresh_planchet_spec, NULL, NULL)) { @@ -181,47 +222,6 @@ run (void *cls, "sent response\n"); continue; } - if (0 == strcmp (op, "create_planchet")) - { - struct TALER_TransferSecretP transfer_secret; - uint32_t coin_num_salt; - struct TALER_PlanchetSecretsP ps; - struct TALER_CoinSpendPublicKeyP coin_pub; - json_t *resp; - struct GNUNET_JSON_Specification eddsa_verify_spec[] = { - GNUNET_JSON_spec_fixed_auto ("transfer_secret", - &transfer_secret), - GNUNET_JSON_spec_uint32 ("coin_index", - &coin_num_salt), - GNUNET_JSON_spec_end () - }; - if (GNUNET_OK != GNUNET_JSON_parse (args, - eddsa_verify_spec, - NULL, - NULL)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "malformed op args\n"); - global_ret = 1; - return; - } - TALER_planchet_setup_refresh (&transfer_secret, - coin_num_salt, &ps); - GNUNET_CRYPTO_eddsa_key_get_public (&ps.coin_priv.eddsa_priv, - &coin_pub.eddsa_pub); - - resp = GNUNET_JSON_PACK ( - GNUNET_JSON_pack_data_auto ("coin_priv", &ps.coin_priv), - GNUNET_JSON_pack_data_auto ("coin_pub", &coin_pub), - GNUNET_JSON_pack_data_auto ("blinding_key", &ps.blinding_key) - ); - json_dumpf (resp, stdout, JSON_COMPACT); - printf ("\n"); - fflush (stdout); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "sent response\n"); - continue; - } GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "unsupported operation '%s'\n", op); From ce28d3b8e4aa513336030468cd08e9cb66718273 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 13:29:07 +0100 Subject: [PATCH 25/37] add latency logging --- src/exchange/taler-exchange-httpd.c | 18 ++++++++++++++++++ src/exchange/taler-exchange-httpd.h | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/src/exchange/taler-exchange-httpd.c b/src/exchange/taler-exchange-httpd.c index 58e9b572a..0ba608ff4 100644 --- a/src/exchange/taler-exchange-httpd.c +++ b/src/exchange/taler-exchange-httpd.c @@ -58,6 +58,11 @@ */ #define UNIX_BACKLOG 50 +/** + * Above what request latency do we start to log? + */ +#define WARN_LATENCY GNUNET_TIME_relative_multiply ( \ + GNUNET_TIME_UNIT_MILLISECONDS, 500) /** * Are clients allowed to request /keys for times other than the @@ -383,6 +388,18 @@ handle_mhd_completion_callback (void *cls, /* Sanity-check that we didn't leave any transactions hanging */ GNUNET_break (GNUNET_OK == TEH_plugin->preflight (TEH_plugin->cls)); + { + struct GNUNET_TIME_Relative latency; + + latency = GNUNET_TIME_absolute_get_duration (rc->start_time); + if (latency.rel_value_us > + WARN_LATENCY.rel_value_us) + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Request for `%s' took %s\n", + rc->url, + GNUNET_STRINGS_relative_time_to_string (latency, + GNUNET_YES)); + } GNUNET_free (rc); *con_cls = NULL; GNUNET_async_scope_restore (&old_scope); @@ -987,6 +1004,7 @@ handle_mhd_request (void *cls, /* We're in a new async scope! */ rc = *con_cls = GNUNET_new (struct TEH_RequestContext); + rc->start_time = GNUNET_TIME_absolute_get (); GNUNET_async_scope_fresh (&rc->async_scope_id); TEH_check_invariants (); rc->url = url; diff --git a/src/exchange/taler-exchange-httpd.h b/src/exchange/taler-exchange-httpd.h index 07f6b0231..cad74d2ed 100644 --- a/src/exchange/taler-exchange-httpd.h +++ b/src/exchange/taler-exchange-httpd.h @@ -218,6 +218,11 @@ struct TEH_RequestContext */ struct GNUNET_AsyncScopeId async_scope_id; + /** + * When was this request started? + */ + struct GNUNET_TIME_Absolute start_time; + /** * Opaque parsing context. */ From a7b4cc97ecfe2452064c8ac2f05b8d7890c8576b Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 14:05:23 +0100 Subject: [PATCH 26/37] minimize idle in transaction from wirewatcher --- src/exchange/taler-exchange-wirewatch.c | 58 +++++++++++++++++-------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/src/exchange/taler-exchange-wirewatch.c b/src/exchange/taler-exchange-wirewatch.c index 6e2cd1ee1..8f69ba7b1 100644 --- a/src/exchange/taler-exchange-wirewatch.c +++ b/src/exchange/taler-exchange-wirewatch.c @@ -125,6 +125,11 @@ struct WireAccount */ bool delay; + /** + * Did we start a transaction yet? + */ + bool started_transaction; + }; @@ -293,7 +298,7 @@ add_account_cb (void *cls, * * @return #GNUNET_OK on success */ -static int +static enum GNUNET_GenericReturnValue exchange_serve_process_config (void) { if (GNUNET_OK != @@ -473,7 +478,7 @@ do_commit (struct WireAccount *wa) * @param json raw JSON response * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration */ -static int +static enum GNUNET_GenericReturnValue history_cb (void *cls, unsigned int http_status, enum TALER_ErrorCode ec, @@ -495,9 +500,13 @@ history_cb (void *cls, (unsigned int) ec, http_status); } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "End of list. Committing progress!\n"); - do_commit (wa); + if (wa->started_transaction) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "End of list. Committing progress!\n"); + wa->started_transaction = false; + do_commit (wa); + } return GNUNET_OK; /* will be ignored anyway */ } if (serial_id < wa->latest_row_off) @@ -507,7 +516,11 @@ history_cb (void *cls, "Serial ID %llu not monotonic (got %llu before). Failing!\n", (unsigned long long) serial_id, (unsigned long long) wa->latest_row_off); - db_plugin->rollback (db_plugin->cls); + if (wa->started_transaction) + { + wa->started_transaction = false; + db_plugin->rollback (db_plugin->cls); + } GNUNET_SCHEDULER_shutdown (); wa->hh = NULL; return GNUNET_SYSERR; @@ -521,10 +534,31 @@ history_cb (void *cls, (unsigned long long) wa->shard_end); wa->latest_row_off = serial_id - 1; wa->delay = false; - do_commit (wa); + if (wa->started_transaction) + { + wa->started_transaction = false; + do_commit (wa); + } + else + GNUNET_break (0); /* how did this happen */ wa->hh = NULL; return GNUNET_SYSERR; } + if (! wa->started_transaction) + { + if (GNUNET_OK != + db_plugin->start_read_committed (db_plugin->cls, + "wirewatch check for incoming wire transfers")) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to start database transaction!\n"); + global_ret = EXIT_FAILURE; + GNUNET_SCHEDULER_shutdown (); + wa->hh = NULL; + return GNUNET_SYSERR; + } + wa->started_transaction = true; + } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Adding wire transfer over %s with (hashed) subject `%s'\n", TALER_amount2s (&details->amount), @@ -645,16 +679,6 @@ find_transfers (void *cls) break; } } - if (GNUNET_OK != - db_plugin->start_read_committed (db_plugin->cls, - "wirewatch check for incoming wire transfers")) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to start database transaction!\n"); - global_ret = EXIT_FAILURE; - GNUNET_SCHEDULER_shutdown (); - return; - } limit = GNUNET_MIN (wa_pos->batch_size, wa_pos->shard_end - wa_pos->batch_start); From ac4247f53a7abcc954feeec1899487d6bd83c01d Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 14:40:06 +0100 Subject: [PATCH 27/37] -note on #7124 --- src/exchange/taler-exchange-wirewatch.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/exchange/taler-exchange-wirewatch.c b/src/exchange/taler-exchange-wirewatch.c index 8f69ba7b1..a417342a2 100644 --- a/src/exchange/taler-exchange-wirewatch.c +++ b/src/exchange/taler-exchange-wirewatch.c @@ -234,6 +234,12 @@ shutdown_task (void *cls) GNUNET_CONTAINER_DLL_remove (wa_head, wa_tail, wa); + if (wa->started_transaction) + { + db_plugin->rollback (db_plugin->cls); + wa->started_transaction = false; + } + // FIXME: delete shard lock here (#7124) GNUNET_free (wa->job_name); GNUNET_free (wa); } @@ -557,6 +563,7 @@ history_cb (void *cls, wa->hh = NULL; return GNUNET_SYSERR; } + wa_pos->shard_start_time = GNUNET_TIME_absolute_get (); wa->started_transaction = true; } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, @@ -669,7 +676,6 @@ find_transfers (void *cls) NULL); return; case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: - wa_pos->shard_start_time = GNUNET_TIME_absolute_get (); wa_pos->shard_start = start; wa_pos->shard_end = end; wa_pos->batch_start = start; From 6c57b33746c318c9ecb8e7c08289d9f1f94ec661 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 16:44:40 +0100 Subject: [PATCH 28/37] -fix warning --- src/exchange/taler-exchange-httpd_db.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/exchange/taler-exchange-httpd_db.c b/src/exchange/taler-exchange-httpd_db.c index 383a7b297..7ced8b88e 100644 --- a/src/exchange/taler-exchange-httpd_db.c +++ b/src/exchange/taler-exchange-httpd_db.c @@ -24,6 +24,7 @@ #include #include "taler_json_lib.h" #include "taler_mhd_lib.h" +#include "taler_exchangedb_lib.h" #include "taler-exchange-httpd_db.h" #include "taler-exchange-httpd_responses.h" From 6dd4a90abdb3883108ae48f48ad73aa94dc64e24 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 22:14:42 +0100 Subject: [PATCH 29/37] fix fakebank issue with timeouts in multi-threaded mode resulting in NPE --- src/bank-lib/fakebank.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/bank-lib/fakebank.c b/src/bank-lib/fakebank.c index e0d8e1556..6549a288d 100644 --- a/src/bank-lib/fakebank.c +++ b/src/bank-lib/fakebank.c @@ -480,11 +480,14 @@ lp_trigger (struct LongPoller *lp, MHD_resume_connection (lp->conn); GNUNET_free (lp); h->mhd_again = true; - if (NULL != h->mhd_task) - GNUNET_SCHEDULER_cancel (h->mhd_task); - h->mhd_task = - GNUNET_SCHEDULER_add_now (&run_mhd, - h); + if (-1 != h->lp_event) + { + if (NULL != h->mhd_task) + GNUNET_SCHEDULER_cancel (h->mhd_task); + h->mhd_task = + GNUNET_SCHEDULER_add_now (&run_mhd, + h); + } } @@ -2413,6 +2416,7 @@ schedule_httpd (struct TALER_FAKEBANK_Handle *h) MHD_UNSIGNED_LONG_LONG timeout; struct GNUNET_TIME_Relative tv; + GNUNET_assert (-1 != h->mhd_fd); haveto = MHD_get_timeout (h->mhd_bank, &timeout); if (MHD_YES == haveto) @@ -2450,6 +2454,7 @@ schedule_httpd (struct TALER_FAKEBANK_Handle *h) MHD_UNSIGNED_LONG_LONG timeout; struct GNUNET_TIME_Relative tv; + GNUNET_assert (-1 == h->lp_event); FD_ZERO (&rs); FD_ZERO (&ws); FD_ZERO (&es); @@ -2521,6 +2526,7 @@ run_mhd (void *cls) h->mhd_again = false; MHD_run (h->mhd_bank); } + GNUNET_assert (-1 == h->lp_event); schedule_httpd (h); } @@ -2554,6 +2560,7 @@ TALER_FAKEBANK_start2 (uint16_t port, GNUNET_assert (strlen (currency) < TALER_CURRENCY_LEN); h = GNUNET_new (struct TALER_FAKEBANK_Handle); h->lp_event = -1; + h->mhd_fd = -1; h->port = port; h->ram_limit = ram_limit; h->serial_counter = 0; From 889625a90f97a23048b3c9dad418f86acb81314b Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 22:15:30 +0100 Subject: [PATCH 30/37] fix idle transaction issue introduced earlier --- src/exchange/taler-exchange-wirewatch.c | 179 +++++++++++++++--------- 1 file changed, 109 insertions(+), 70 deletions(-) diff --git a/src/exchange/taler-exchange-wirewatch.c b/src/exchange/taler-exchange-wirewatch.c index a417342a2..eb0449942 100644 --- a/src/exchange/taler-exchange-wirewatch.c +++ b/src/exchange/taler-exchange-wirewatch.c @@ -362,6 +362,7 @@ static void handle_soft_error (struct WireAccount *wa) { db_plugin->rollback (db_plugin->cls); + wa->started_transaction = false; if (1 < wa->batch_size) { wa->batch_thresh = wa->batch_size; @@ -376,6 +377,89 @@ handle_soft_error (struct WireAccount *wa) } +/** + * We are done with a shard, move on to the next one. + * + * @param wa wire account for which we completed a shard + */ +static void +shard_completed (struct WireAccount *wa) +{ + /* transaction success, update #last_row_off */ + wa->batch_start = wa->latest_row_off; + if (wa->batch_size < MAXIMUM_BATCH_SIZE) + { + int delta; + + delta = ((int) wa->batch_thresh - (int) wa->batch_size) / 4; + if (delta < 0) + delta = -delta; + wa->batch_size = GNUNET_MIN (MAXIMUM_BATCH_SIZE, + wa->batch_size + delta + 1); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Increasing batch size to %llu\n", + (unsigned long long) wa->batch_size); + } + if (wa->delay) + { + wa->delayed_until + = GNUNET_TIME_relative_to_absolute (wirewatch_idle_sleep_interval); + wa_pos = wa_pos->next; + if (NULL == wa_pos) + wa_pos = wa_head; + GNUNET_assert (NULL != wa_pos); + } + GNUNET_assert (NULL == task); + task = GNUNET_SCHEDULER_add_at (wa_pos->delayed_until, + &find_transfers, + NULL); +} + + +/** + * We are finished with the current shard. Update the database, marking the + * shard as finished. + * + * @param wa wire account to commit for + * @return true on success + */ +static bool +mark_shard_done (struct WireAccount *wa) +{ + enum GNUNET_DB_QueryStatus qs; + + if (wa->shard_end > wa->latest_row_off) + return false; /* actually, not done! */ + /* shard is complete, mark this as well */ + qs = db_plugin->complete_shard (db_plugin->cls, + wa->job_name, + wa->shard_start, + wa->shard_end); + switch (qs) + { + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + db_plugin->rollback (db_plugin->cls); + GNUNET_SCHEDULER_shutdown (); + return false; + case GNUNET_DB_STATUS_SOFT_ERROR: + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Got DB soft error for complete_shard. Rolling back.\n"); + handle_soft_error (wa); + return false; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + /* already existed, ok, let's just continue */ + break; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + /* normal case */ + shard_delay = GNUNET_TIME_absolute_get_duration (wa->shard_start_time); + + break; + } + return true; +} + + /** * We are finished with the current transaction, try * to commit and then schedule the next iteration. @@ -387,35 +471,8 @@ do_commit (struct WireAccount *wa) { enum GNUNET_DB_QueryStatus qs; - if (wa->shard_end <= wa->latest_row_off) - { - /* shard is complete, mark this as well */ - qs = db_plugin->complete_shard (db_plugin->cls, - wa->job_name, - wa->shard_start, - wa->shard_end); - switch (qs) - { - case GNUNET_DB_STATUS_HARD_ERROR: - GNUNET_break (0); - db_plugin->rollback (db_plugin->cls); - GNUNET_SCHEDULER_shutdown (); - return; - case GNUNET_DB_STATUS_SOFT_ERROR: - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Got DB soft error for complete_shard. Rolling back.\n"); - handle_soft_error (wa); - return; - case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: - /* already existed, ok, let's just continue */ - break; - case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: - /* normal case */ - shard_delay = GNUNET_TIME_absolute_get_duration (wa->shard_start_time); - - break; - } - } + wa->started_transaction = false; + mark_shard_done (wa); qs = db_plugin->commit (db_plugin->cls); switch (qs) { @@ -432,43 +489,7 @@ do_commit (struct WireAccount *wa) /* normal case */ break; } - /* transaction success, update #last_row_off */ - wa->batch_start = wa->latest_row_off; - if (wa->batch_size < MAXIMUM_BATCH_SIZE) - { - int delta; - - delta = ((int) wa->batch_thresh - (int) wa->batch_size) / 4; - if (delta < 0) - delta = -delta; - wa->batch_size = GNUNET_MIN (MAXIMUM_BATCH_SIZE, - wa->batch_size + delta + 1); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Increasing batch size to %llu\n", - (unsigned long long) wa->batch_size); - } - if ( (wa->delay) && - (test_mode) && - (NULL == wa->next) ) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Shutdown due to test mode!\n"); - GNUNET_SCHEDULER_shutdown (); - return; - } - if (wa->delay) - { - wa->delayed_until - = GNUNET_TIME_relative_to_absolute (wirewatch_idle_sleep_interval); - wa_pos = wa_pos->next; - if (NULL == wa_pos) - wa_pos = wa_head; - GNUNET_assert (NULL != wa_pos); - } - GNUNET_assert (NULL == task); - task = GNUNET_SCHEDULER_add_at (wa_pos->delayed_until, - &find_transfers, - NULL); + shard_completed (wa); } @@ -510,9 +531,20 @@ history_cb (void *cls, { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "End of list. Committing progress!\n"); - wa->started_transaction = false; do_commit (wa); } + else + { + if ( (wa->delay) && + (test_mode) && + (NULL == wa->next) ) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Shutdown due to test mode!\n"); + GNUNET_SCHEDULER_shutdown (); + return GNUNET_OK; + } + } return GNUNET_OK; /* will be ignored anyway */ } if (serial_id < wa->latest_row_off) @@ -542,11 +574,13 @@ history_cb (void *cls, wa->delay = false; if (wa->started_transaction) { - wa->started_transaction = false; do_commit (wa); } else - GNUNET_break (0); /* how did this happen */ + { + if (mark_shard_done (wa)) + shard_completed (wa); + } wa->hh = NULL; return GNUNET_SYSERR; } @@ -587,6 +621,7 @@ history_cb (void *cls, case GNUNET_DB_STATUS_HARD_ERROR: GNUNET_break (0); db_plugin->rollback (db_plugin->cls); + wa->started_transaction = false; GNUNET_SCHEDULER_shutdown (); wa->hh = NULL; return GNUNET_SYSERR; @@ -703,7 +738,11 @@ find_transfers (void *cls) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to start request for account history!\n"); - db_plugin->rollback (db_plugin->cls); + if (wa_pos->started_transaction) + { + db_plugin->rollback (db_plugin->cls); + wa_pos->started_transaction = false; + } global_ret = EXIT_FAILURE; GNUNET_SCHEDULER_shutdown (); return; From fba91c63d57d73732249b972127575ca1fd4d5ff Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 23:13:39 +0100 Subject: [PATCH 31/37] introduce stored procedure for coin balance check --- src/exchange/taler-exchange-httpd_db.c | 110 +++++++++++- src/exchange/taler-exchange-httpd_db.h | 3 + src/exchangedb/exchange-0001.sql | 188 ++++++++++++++++++++ src/exchangedb/plugin_exchangedb_postgres.c | 59 +++++- src/include/taler_exchangedb_plugin.h | 25 ++- 5 files changed, 367 insertions(+), 18 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_db.c b/src/exchange/taler-exchange-httpd_db.c index 7ced8b88e..388679c38 100644 --- a/src/exchange/taler-exchange-httpd_db.c +++ b/src/exchange/taler-exchange-httpd_db.c @@ -164,14 +164,29 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin, } -enum GNUNET_DB_QueryStatus -TEH_check_coin_balance (struct MHD_Connection *connection, - const struct TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_Amount *coin_value, - const struct TALER_Amount *op_cost, - bool check_recoup, - bool zombie_required, - MHD_RESULT *mhd_ret) +/** + * Called when we actually know that the balance (was) insufficient. + * Re-does the check (slowly) to compute the full error message for + * the client. + * + * @param connection HTTP connection to report hard errors on + * @param coin_pub coin to analyze + * @param coin_value total value of the original coin (by denomination) + * @param op_cost cost of the current operation (for error reporting) + * @param check_recoup should we include recoup transactions in the check + * @param zombie_required additional requirement that the coin must + * be a zombie coin, or also hard failure + * @param[out] mhd_ret set to response status code, on hard error only + * @return transaction status + */ +static enum GNUNET_DB_QueryStatus +check_coin_balance (struct MHD_Connection *connection, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *coin_value, + const struct TALER_Amount *op_cost, + bool check_recoup, + bool zombie_required, + MHD_RESULT *mhd_ret) { struct TALER_EXCHANGEDB_TransactionList *tl; struct TALER_Amount spent; @@ -273,13 +288,90 @@ TEH_check_coin_balance (struct MHD_Connection *connection, return GNUNET_DB_STATUS_HARD_ERROR; } - /* we're good, coin has sufficient funds to be melted */ + /* This should not happen: The coin has sufficient funds + after all!?!? */ TEH_plugin->free_coin_transaction_list (TEH_plugin->cls, tl); return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; } +enum GNUNET_DB_QueryStatus +TEH_check_coin_balance (struct MHD_Connection *connection, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *coin_value, + const struct TALER_Amount *op_cost, + bool check_recoup, + bool zombie_required, + MHD_RESULT *mhd_ret) +{ + bool balance_ok = false; + bool zombie_ok = false; + enum GNUNET_DB_QueryStatus qs; + + qs = TEH_plugin->do_check_coin_balance (TEH_plugin->cls, + coin_pub, + coin_value, + check_recoup, + zombie_required, + &balance_ok, + &zombie_ok); + switch (qs) + { + case GNUNET_DB_STATUS_HARD_ERROR: + *mhd_ret = TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "check_coin_balance"); + return qs; + case GNUNET_DB_STATUS_SOFT_ERROR: + return qs; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + GNUNET_break (0); + *mhd_ret = TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "check_coin_balance"); + return GNUNET_DB_STATUS_HARD_ERROR; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + /* handled below */ + break; + } + if (! zombie_ok) + { + GNUNET_break_op (0); + *mhd_ret = TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_BAD_REQUEST, + TALER_EC_EXCHANGE_MELT_COIN_EXPIRED_NO_ZOMBIE, + NULL); + return GNUNET_DB_STATUS_HARD_ERROR; + } + if (balance_ok) + return qs; + /* balance is not OK, do expensive call to compute full error message */ + qs = check_coin_balance (connection, + coin_pub, + coin_value, + op_cost, + check_recoup, + zombie_required, + mhd_ret); + if (qs < 0) + return qs; /* we expected to fail (same check as before!) */ + GNUNET_break (0); /* stored procedure and individual statements + disagree, should be impossible! */ + *mhd_ret = TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_INVARIANT_FAILURE, + "stored procedure disagrees with full coin transaction history fetch"); + return GNUNET_DB_STATUS_HARD_ERROR; +} + + enum GNUNET_GenericReturnValue TEH_DB_run_transaction (struct MHD_Connection *connection, const char *name, diff --git a/src/exchange/taler-exchange-httpd_db.h b/src/exchange/taler-exchange-httpd_db.h index 60885dbd1..5ee3b41d5 100644 --- a/src/exchange/taler-exchange-httpd_db.h +++ b/src/exchange/taler-exchange-httpd_db.h @@ -47,6 +47,9 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin, * insufficient for all transactions associated with the * coin, return a hard error. * + * We first do a "fast" check using a stored procedure, and + * only obtain the "full" data on failure (for performance). + * * @param connection HTTP connection to report hard errors on * @param coin_pub coin to analyze * @param coin_value total value of the original coin (by denomination) diff --git a/src/exchangedb/exchange-0001.sql b/src/exchangedb/exchange-0001.sql index dc6b2bba2..1725b70e0 100644 --- a/src/exchangedb/exchange-0001.sql +++ b/src/exchangedb/exchange-0001.sql @@ -899,6 +899,194 @@ COMMENT ON FUNCTION exchange_do_withdraw_limit_check(INT8, INT8, INT8, INT4) +CREATE OR REPLACE FUNCTION exchange_do_check_coin_balance( + IN denom_val INT8, -- value of the denomination of the coin + IN denom_frac INT4, -- value of the denomination of the coin + IN in_coin_pub BYTEA, -- coin public key + IN check_recoup BOOLEAN, -- do we need to check the recoup table? + IN zombie_required BOOLEAN, -- do we need a zombie coin? + OUT balance_ok BOOLEAN, -- balance satisfied? + OUT zombie_ok BOOLEAN) -- zombie satisfied? +LANGUAGE plpgsql +AS $$ +DECLARE + coin_uuid INT8; -- known_coin_id of coin_pub +DECLARE + tmp_val INT8; -- temporary result +DECLARE + tmp_frac INT8; -- temporary result +DECLARE + spent_val INT8; -- how much of coin was spent? +DECLARE + spent_frac INT8; -- how much of coin was spent? +DECLARE + unspent_val INT8; -- how much of coin was refunded? +DECLARE + unspent_frac INT8; -- how much of coin was refunded? +BEGIN + +-- Note: possible future optimization: get the coin_uuid from the previous +-- 'ensure_coin_known' and pass that here instead of the coin_pub. Might help +-- a tiny bit with performance. +SELECT known_coin_id INTO coin_uuid + FROM known_coins + WHERE coin_pub=in_coin_pub; + +IF NOT FOUND +THEN + -- coin unknown, should be impossible! + balance_ok=FALSE; + zombie_ok=FALSE; + ASSERT false, 'coin unknown'; + RETURN; +END IF; + + +spent_val = 0; +spent_frac = 0; +unspent_val = denom_val; +unspent_frac = denom_frac; + +SELECT + SUM(amount_with_fee_val) -- overflow here is not plausible + ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM deposits + WHERE known_coin_id=coin_uuid; + +IF tmp_val IS NOT NULL +THEN + spent_val = spent_val + tmp_val; + spent_frac = spent_frac + tmp_frac; +END IF; + +SELECT + SUM(amount_with_fee_val) -- overflow here is not plausible + ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM refresh_commitments + WHERE old_known_coin_id=coin_uuid; + +IF tmp_val IS NOT NULL +THEN + spent_val = spent_val + tmp_val; + spent_frac = spent_frac + tmp_frac; +END IF; + + +SELECT + SUM(rf.amount_with_fee_val) -- overflow here is not plausible + ,SUM(CAST(rf.amount_with_fee_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM deposits + JOIN refunds rf + USING (deposit_serial_id) + WHERE + known_coin_id=coin_uuid; +IF tmp_val IS NOT NULL +THEN + unspent_val = unspent_val + tmp_val; + unspent_frac = unspent_frac + tmp_frac; +END IF; + +-- Note: even if 'check_recoup' is true, the tables below +-- are in practice likely empty (as they only apply if +-- the exchange (ever) had to revoke keys). +IF check_recoup +THEN + + SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup_refresh + WHERE known_coin_id=coin_uuid; + + IF tmp_val IS NOT NULL + THEN + spent_val = spent_val + tmp_val; + spent_frac = spent_frac + tmp_frac; + END IF; + + SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup + WHERE known_coin_id=coin_uuid; + + IF tmp_val IS NOT NULL + THEN + spent_val = spent_val + tmp_val; + spent_frac = spent_frac + tmp_frac; + END IF; + + SELECT + SUM(amount_val) -- overflow here is not plausible + ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits + INTO + tmp_val + ,tmp_frac + FROM recoup_refresh + JOIN refresh_revealed_coins rrc + USING (rrc_serial) + JOIN refresh_commitments rfc + ON (rrc.melt_serial_id = rfc.melt_serial_id) + WHERE rfc.old_known_coin_id=coin_uuid; + + IF tmp_val IS NOT NULL + THEN + unspent_val = unspent_val + tmp_val; + unspent_frac = unspent_frac + tmp_frac; + END IF; + + IF ( (0 < tmp_val) OR (0 < tmp_frac) ) + THEN + -- There was a transaction that justifies the zombie + -- status, clear the flag + zombie_required=FALSE; + END IF; + +END IF; + + +-- normalize results +spent_val = spent_val + spent_frac / 100000000; +spent_frac = spent_frac % 100000000; +unspent_val = unspent_val + unspent_frac / 100000000; +unspent_frac = unspent_frac % 100000000; + +-- Actually check if the coin balance is sufficient. Verbosely. ;-) +IF (unspent_val > spent_val) +THEN + balance_ok=TRUE; +ELSE + IF (unspent_val = spent_val) AND (unspent_frac >= spent_frac) + THEN + balance_ok=TRUE; + ELSE + balance_ok=FALSE; + END IF; +END IF; + +zombie_ok = NOT zombie_required; + +END $$; + +COMMENT ON FUNCTION exchange_do_check_coin_balance(INT8, INT4, BYTEA, BOOLEAN, BOOLEAN) + IS 'Checks whether the coin has sufficient balance for all the operations associated with it'; + + -- Complete transaction diff --git a/src/exchangedb/plugin_exchangedb_postgres.c b/src/exchangedb/plugin_exchangedb_postgres.c index 73a03e751..8e184a9dd 100644 --- a/src/exchangedb/plugin_exchangedb_postgres.c +++ b/src/exchangedb/plugin_exchangedb_postgres.c @@ -596,6 +596,16 @@ prepare_statements (struct PostgresClosure *pg) "lock_withdraw", "LOCK TABLE reserves_out;", 0), + /* Used in #postgres_do_check_coin_balance() to check + a coin's balance */ + GNUNET_PQ_make_prepare ( + "call_check_coin_balance", + "SELECT " + " balance_ok" + ",zombie_ok" + " FROM exchange_do_check_coin_balance" + " ($1,$2,$3,$4,$5);", + 5), /* Used in #postgres_do_withdraw() to store the signature of a blinded coin with the blinded coin's details before returning it during /reserve/withdraw. We store @@ -4491,6 +4501,53 @@ postgres_get_withdraw_info ( } +/** + * Check coin balance is sufficient to satisfy balance + * invariants. + * + * @param cls the `struct PostgresClosure` with the plugin-specific state + * @param coin_pub coin to check + * @param coin_value value of the coin's denomination (avoids internal lookup) + * @param check_recoup include recoup and recoup_refresh tables in calculation + * @param zombie_required additionally require coin to be a zombie coin + * @param[out] balance_ok set to true if the balance was sufficient + * @param[out] zombie_ok set to true if the zombie requirement was satisfied + * @return query execution status + */ +static enum GNUNET_DB_QueryStatus +postgres_do_check_coin_balance ( + void *cls, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *coin_value, + bool check_recoup, + bool zombie_required, + bool *balance_ok, + bool *zombie_ok) +{ + struct PostgresClosure *pg = cls; + struct GNUNET_PQ_QueryParam params[] = { + TALER_PQ_query_param_amount (coin_value), + GNUNET_PQ_query_param_auto_from_type (coin_pub), + GNUNET_PQ_query_param_bool (check_recoup), + GNUNET_PQ_query_param_bool (zombie_required), + GNUNET_PQ_query_param_end + }; + struct GNUNET_PQ_ResultSpec rs[] = { + GNUNET_PQ_result_spec_bool ("balance_ok", + balance_ok), + GNUNET_PQ_result_spec_bool ("zombie_ok", + zombie_ok), + GNUNET_PQ_result_spec_end + }; + + return GNUNET_PQ_eval_prepared_singleton_select (pg->conn, + "call_check_coin_balance", + params, + rs); + +} + + /** * Perform withdraw operation, checking for sufficient balance * and possibly persisting the withdrawal details. @@ -11825,7 +11882,7 @@ libtaler_plugin_exchangedb_postgres_init (void *cls) plugin->get_latest_reserve_in_reference = &postgres_get_latest_reserve_in_reference; plugin->get_withdraw_info = &postgres_get_withdraw_info; - // plugin->insert_withdraw_info = &postgres_insert_withdraw_info; + plugin->do_check_coin_balance = &postgres_do_check_coin_balance; plugin->do_withdraw = &postgres_do_withdraw; plugin->do_withdraw_limit_check = &postgres_do_withdraw_limit_check; plugin->get_reserve_history = &postgres_get_reserve_history; diff --git a/src/include/taler_exchangedb_plugin.h b/src/include/taler_exchangedb_plugin.h index fd2f3dc42..56a16dd72 100644 --- a/src/include/taler_exchangedb_plugin.h +++ b/src/include/taler_exchangedb_plugin.h @@ -2503,18 +2503,27 @@ struct TALER_EXCHANGEDB_Plugin /** - * Store collectable coin under the corresponding hash of the blinded - * message. + * Check coin balance is sufficient to satisfy balance + * invariants. * - * @param cls the @e cls of this struct with the plugin-specific state - * @param collectable corresponding collectable coin (blind signature) - * if a coin is found - * @return statement execution status + * @param cls the `struct PostgresClosure` with the plugin-specific state + * @param coin_pub coin to check + * @param coin_value value of the coin's denomination (avoids internal lookup) + * @param check_recoup include recoup and recoup_refresh tables in calculation + * @param zombie_required additionally require coin to be a zombie coin + * @param[out] balance_ok set to true if the balance was sufficient + * @param[out] zombie_ok set to true if the zombie requirement was satisfied + * @return query execution status */ enum GNUNET_DB_QueryStatus - (*insert_withdraw_infoXX)( + (*do_check_coin_balance)( void *cls, - const struct TALER_EXCHANGEDB_CollectableBlindcoin *collectable); + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *coin_value, + bool check_recoup, + bool zombie_required, + bool *balance_ok, + bool *zombie_ok); /** From 4c8b99f333064b3c0801aa926a74d155b1e22a5f Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 23:16:55 +0100 Subject: [PATCH 32/37] -note FIXME --- src/exchange/taler-exchange-httpd_recoup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/exchange/taler-exchange-httpd_recoup.c b/src/exchange/taler-exchange-httpd_recoup.c index fd8baf65d..24f1073e9 100644 --- a/src/exchange/taler-exchange-httpd_recoup.c +++ b/src/exchange/taler-exchange-httpd_recoup.c @@ -101,6 +101,7 @@ struct RecoupContext }; +// FIXME: this code should be simplified by using TEH_check_coin_balance() /** * Execute a "recoup". The validity of the coin and signature have * already been checked. The database must now check that the coin is From 386301d1cf48ad3c87df1733cd7a82957ff6c383 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 9 Dec 2021 23:17:15 +0100 Subject: [PATCH 33/37] -was integrated --- src/exchangedb/melt.sql | 171 ---------------------------------------- 1 file changed, 171 deletions(-) delete mode 100644 src/exchangedb/melt.sql diff --git a/src/exchangedb/melt.sql b/src/exchangedb/melt.sql deleted file mode 100644 index fc6d24d45..000000000 --- a/src/exchangedb/melt.sql +++ /dev/null @@ -1,171 +0,0 @@ - --- Everything in one big transaction --- BEGIN; - --- Check patch versioning is in place. --- SELECT _v.register_patch('exchange-000x', NULL, NULL); - - -CREATE OR REPLACE FUNCTION exchange_check_coin_balance( - IN denom_val INT8, -- value of the denomination of the coin - IN denom_frac INT4, -- value of the denomination of the coin - IN in_coin_pub BYTEA, -- coin public key - IN check_recoup BOOLEAN, -- do we need to check the recoup table? - IN zombie_required BOOLEAN, -- do we need a zombie coin? - OUT balance_ok BOOLEAN, -- balance satisfied? - OUT zombie_ok BOOLEAN) -- zombie satisfied? -LANGUAGE plpgsql -AS $$ -DECLARE - coin_uuid INT8; -- known_coin_id of coin_pub -DECLARE - tmp_val INT8; -- temporary result -DECLARE - tmp_frac INT8; -- temporary result -DECLARE - spent_val INT8; -- how much of coin was spent? -DECLARE - spent_frac INT8; -- how much of coin was spent? -DECLARE - unspent_val INT8; -- how much of coin was refunded? -DECLARE - unspent_frac INT8; -- how much of coin was refunded? -BEGIN - --- Note: possible future optimization: get the coin_uuid from the previous --- 'ensure_coin_known' and pass that here instead of the coin_pub. Might help --- a tiny bit with performance. -SELECT known_coin_id INTO coin_uuid - FROM known_coins - WHERE coin_pub=in_coin_pub; - -IF NOT FOUND -THEN - -- coin unknown, should be impossible! - balance_ok=FALSE; - zombie_ok=FALSE; - ASSERT false, 'coin unknown'; - RETURN; -END IF; - - -spent_val = 0; -spent_frac = 0; -unspent_val = denom_val; -unspent_frac = denom_frac; - -SELECT - SUM(amount_with_fee_val) -- overflow here is not plausible - ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM deposits - WHERE known_coin_id=coin_uuid; - -spent_val = spent_val + tmp_val; -spent_frac = spent_frac + tmp_frac; - -SELECT - SUM(amount_with_fee_val) -- overflow here is not plausible - ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM refresh_commitments - WHERE old_known_coin_id=coin_uuid; - -spent_val = spent_val + tmp_val; -spent_frac = spent_frac + tmp_frac; - -SELECT - SUM(amount_with_fee_val) -- overflow here is not plausible - ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM refunds - WHERE known_coin_id=coin_uuid; - -unspent_val = unspent_val + tmp_val; -unspent_frac = unspent_frac + tmp_frac; - --- Note: even if 'check_recoup' is true, the tables below --- are in practice likely empty (as they only apply if --- the exchange (ever) had to revoke keys). -IF check_recoup -THEN - - SELECT - SUM(amount_val) -- overflow here is not plausible - ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM recoup_refresh - WHERE known_coin_id=coin_uuid; - - unspent_val = unspent_val + tmp_val; - unspent_frac = unspent_frac + tmp_frac; - - SELECT - SUM(amount_val) -- overflow here is not plausible - ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM recoup - WHERE known_coin_id=coin_uuid; - - spent_val = spent_val + tmp_val; - spent_frac = spent_frac + tmp_frac; - - SELECT - SUM(amount_val) -- overflow here is not plausible - ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits - INTO - tmp_val - ,tmp_frac - FROM recoup_refresh - JOIN refresh_revealed_coins rrc - USING (rrc_serial) - JOIN refresh_commitments rfc - ON (rrc.melt_serial_id = rfc.melt_serial_id) - WHERE rfc.old_known_coin_id=coin_uuid; - - spent_val = spent_val + tmp_val; - spent_frac = spent_frac + tmp_frac; - - IF ( (0 < tmp_val) OR (0 < tmp_frac) ) - THEN - -- There was a transaction that justifies the zombie - -- status, clear the flag - zombie_required=FALSE; - END IF; - -END IF; - - --- Actually check if the coin balance is sufficient. Verbosely. ;-) -IF (unspent_val > spent_val) -THEN - balance_ok=TRUE; -ELSE - IF (reserve_val == amount_val) AND (reserve_frac >= amount_frac) - THEN - balance_ok=TRUE; - ELSE - balance_ok=FALSE; - END IF; -END IF; - -zombie_ok = NOT zombie_required; - -END $$; - -COMMENT ON FUNCTION exchange_check_coin_balance(INT8, INT4, BYTEA, BOOLEAN, BOOLEAN) - IS 'Checks whether the coin has sufficient balance for all the operations associated with it'; - - --- Complete transaction --- COMMIT; From 4eb958846fdd91f3f912665bd0c0295e8eebcf1d Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 11 Dec 2021 14:03:08 +0100 Subject: [PATCH 34/37] -more logging, less hanging ;-) --- src/exchange/taler-exchange-wirewatch.c | 31 +++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/exchange/taler-exchange-wirewatch.c b/src/exchange/taler-exchange-wirewatch.c index eb0449942..6b63de76a 100644 --- a/src/exchange/taler-exchange-wirewatch.c +++ b/src/exchange/taler-exchange-wirewatch.c @@ -410,6 +410,11 @@ shard_completed (struct WireAccount *wa) GNUNET_assert (NULL != wa_pos); } GNUNET_assert (NULL == task); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Will look for more transfers in %s\n", + GNUNET_STRINGS_relative_time_to_string ( + GNUNET_TIME_absolute_get_remaining (wa_pos->delayed_until), + GNUNET_YES)); task = GNUNET_SCHEDULER_add_at (wa_pos->delayed_until, &find_transfers, NULL); @@ -527,6 +532,11 @@ history_cb (void *cls, (unsigned int) ec, http_status); } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "History response complete\n"); + } if (wa->started_transaction) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, @@ -544,6 +554,10 @@ history_cb (void *cls, GNUNET_SCHEDULER_shutdown (); return GNUNET_OK; } + else + { + shard_completed (wa); + } } return GNUNET_OK; /* will be ignored anyway */ } @@ -700,12 +714,22 @@ find_transfers (void *cls) return; case GNUNET_DB_STATUS_SOFT_ERROR: /* try again */ + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Serialization error tying to obtain shard, will try again in %s!\n", + GNUNET_STRINGS_relative_time_to_string ( + wirewatch_idle_sleep_interval, + GNUNET_YES)); task = GNUNET_SCHEDULER_add_delayed (wirewatch_idle_sleep_interval, &find_transfers, NULL); return; case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: GNUNET_break (0); + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "No shard available, will try again in %s!\n", + GNUNET_STRINGS_relative_time_to_string ( + wirewatch_idle_sleep_interval, + GNUNET_YES)); task = GNUNET_SCHEDULER_add_delayed (wirewatch_idle_sleep_interval, &find_transfers, NULL); @@ -715,8 +739,11 @@ find_transfers (void *cls) wa_pos->shard_end = end; wa_pos->batch_start = start; GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Starting with shard at %llu\n", - (unsigned long long) start); + "Starting with shard at [%llu,%llu) locked for %s\n", + (unsigned long long) start, + (unsigned long long) end, + GNUNET_STRINGS_relative_time_to_string (delay, + GNUNET_YES)); break; } } From 40175318e9a824d01ba9d592b92f244027cc618b Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 11 Dec 2021 15:39:24 +0100 Subject: [PATCH 35/37] towards simplified / more efficient /recoup --- src/exchange/taler-exchange-httpd_recoup.c | 123 ++++++++++----------- 1 file changed, 61 insertions(+), 62 deletions(-) diff --git a/src/exchange/taler-exchange-httpd_recoup.c b/src/exchange/taler-exchange-httpd_recoup.c index 24f1073e9..c2cae0861 100644 --- a/src/exchange/taler-exchange-httpd_recoup.c +++ b/src/exchange/taler-exchange-httpd_recoup.c @@ -1,6 +1,6 @@ /* This file is part of TALER - Copyright (C) 2017-2020 Taler Systems SA + Copyright (C) 2017-2021 Taler Systems SA TALER is free software; you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software @@ -28,6 +28,7 @@ #include #include "taler_json_lib.h" #include "taler_mhd_lib.h" +#include "taler-exchange-httpd_db.h" #include "taler-exchange-httpd_recoup.h" #include "taler-exchange-httpd_responses.h" #include "taler-exchange-httpd_keys.h" @@ -94,9 +95,9 @@ struct RecoupContext struct GNUNET_TIME_Absolute now; /** - * #GNUNET_YES if the client claims the coin originated from a refresh. + * true if the client claims the coin originated from a refresh. */ - int refreshed; + bool refreshed; }; @@ -129,57 +130,10 @@ recoup_transaction (void *cls, struct TALER_Amount spent; struct TALER_Amount recouped; enum GNUNET_DB_QueryStatus qs; - int existing_recoup_found; + bool existing_recoup_found; /* Check whether a recoup is allowed, and if so, to which reserve / account the money should go */ - if (pc->refreshed) - { - qs = TEH_plugin->get_old_coin_by_h_blind (TEH_plugin->cls, - &pc->h_blind, - &pc->target.old_coin_pub); - if (0 > qs) - { - if (GNUNET_DB_STATUS_HARD_ERROR == qs) - { - GNUNET_break (0); - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_FETCH_FAILED, - "old coin by h_blind"); - } - return qs; - } - } - else - { - qs = TEH_plugin->get_reserve_by_h_blind (TEH_plugin->cls, - &pc->h_blind, - &pc->target.reserve_pub); - if (0 > qs) - { - if (GNUNET_DB_STATUS_HARD_ERROR == qs) - { - GNUNET_break (0); - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_FETCH_FAILED, - "reserve by h_blind"); - } - return qs; - } - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Recoup requested for unknown envelope %s\n", - GNUNET_h2s (&pc->h_blind.hash)); - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_NOT_FOUND, - TALER_EC_EXCHANGE_RECOUP_WITHDRAW_NOT_FOUND, - NULL); - return GNUNET_DB_STATUS_HARD_ERROR; - } /* Calculate remaining balance, including recoups already applied. */ qs = TEH_plugin->get_coin_transactions (TEH_plugin->cls, @@ -206,7 +160,7 @@ recoup_transaction (void *cls, TALER_amount_set_zero (pc->value.currency, &recouped)); /* Check if this coin has been recouped already at least once */ - existing_recoup_found = GNUNET_NO; + existing_recoup_found = false; for (struct TALER_EXCHANGEDB_TransactionList *pos = tl; NULL != pos; pos = pos->next) @@ -214,7 +168,7 @@ recoup_transaction (void *cls, if ( (TALER_EXCHANGEDB_TT_RECOUP == pos->type) || (TALER_EXCHANGEDB_TT_RECOUP_REFRESH == pos->type) ) { - existing_recoup_found = GNUNET_YES; + existing_recoup_found = true; break; } } @@ -252,8 +206,7 @@ recoup_transaction (void *cls, NULL); return GNUNET_DB_STATUS_HARD_ERROR; } - if ( (0 == pc->amount.fraction) && - (0 == pc->amount.value) ) + if (TALER_amount_is_zero (&pc->amount)) { /* Recoup has no effect: coin fully spent! */ enum GNUNET_DB_QueryStatus ret; @@ -332,7 +285,7 @@ recoup_transaction (void *cls, * @param coin information about the coin * @param coin_bks blinding data of the coin (to be checked) * @param coin_sig signature of the coin - * @param refreshed #GNUNET_YES if the coin was refreshed + * @param refreshed true if the coin was refreshed * @return MHD result code */ static MHD_RESULT @@ -341,7 +294,7 @@ verify_and_execute_recoup ( const struct TALER_CoinPublicInfo *coin, const union TALER_DenominationBlindingKeyP *coin_bks, const struct TALER_CoinSpendSignatureP *coin_sig, - int refreshed) + bool refreshed) { struct RecoupContext pc; const struct TEH_DenominationKey *dk; @@ -460,7 +413,6 @@ verify_and_execute_recoup ( GNUNET_free (coin_ev); } - /* Perform actual recoup transaction */ pc.coin_sig = coin_sig; pc.coin_bks = coin_bks; pc.coin = coin; @@ -480,6 +432,53 @@ verify_and_execute_recoup ( return mhd_ret; } + { + enum GNUNET_DB_QueryStatus qs; + + if (pc.refreshed) + { + qs = TEH_plugin->get_old_coin_by_h_blind (TEH_plugin->cls, + &pc.h_blind, + &pc.target.old_coin_pub); + if (0 > qs) + { + GNUNET_break (0); + return TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "old coin by h_blind"); + } + } + else + { + qs = TEH_plugin->get_reserve_by_h_blind (TEH_plugin->cls, + &pc.h_blind, + &pc.target.reserve_pub); + if (0 > qs) + { + GNUNET_break (0); + return TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "reserve by h_blind"); + } + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Recoup requested for unknown envelope %s\n", + GNUNET_h2s (&pc.h_blind.hash)); + return TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_NOT_FOUND, + TALER_EC_EXCHANGE_RECOUP_WITHDRAW_NOT_FOUND, + NULL); + } + } + + /* Perform actual recoup transaction */ { MHD_RESULT mhd_ret; @@ -531,7 +530,7 @@ TEH_handler_recoup (struct MHD_Connection *connection, struct TALER_CoinPublicInfo coin; union TALER_DenominationBlindingKeyP coin_bks; struct TALER_CoinSpendSignatureP coin_sig; - int refreshed = GNUNET_NO; + bool refreshed = false; struct GNUNET_JSON_Specification spec[] = { GNUNET_JSON_spec_fixed_auto ("denom_pub_hash", &coin.denom_pub_hash), @@ -541,9 +540,9 @@ TEH_handler_recoup (struct MHD_Connection *connection, &coin_bks), GNUNET_JSON_spec_fixed_auto ("coin_sig", &coin_sig), - GNUNET_JSON_spec_mark_optional - (GNUNET_JSON_spec_boolean ("refreshed", - &refreshed)), + GNUNET_JSON_spec_mark_optional ( + GNUNET_JSON_spec_bool ("refreshed", + &refreshed)), GNUNET_JSON_spec_end () }; From a25429cadd29177f2bfb56f644696b2cbfbadf72 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 11 Dec 2021 20:06:15 +0100 Subject: [PATCH 36/37] fix #7123 --- src/curl/curl.c | 13 --- src/lib/exchange_api_common.c | 17 ---- src/lib/exchange_api_curl_defaults.c | 14 +-- src/lib/exchange_api_handle.c | 139 +-------------------------- src/lib/exchange_api_link.c | 19 ---- 5 files changed, 8 insertions(+), 194 deletions(-) diff --git a/src/curl/curl.c b/src/curl/curl.c index 424c41fd6..5009fa3cf 100644 --- a/src/curl/curl.c +++ b/src/curl/curl.c @@ -30,14 +30,6 @@ #endif -/** - * Add the @a body as POST data to the easy handle in @a ctx. - * - * @param[in,out] ctx a request context (updated) - * @param eh easy handle to use - * @param body JSON body to add to @e ctx - * @return #GNUNET_OK on success #GNUNET_SYSERR on failure - */ enum GNUNET_GenericReturnValue TALER_curl_easy_post (struct TALER_CURL_PostContext *ctx, CURL *eh, @@ -101,11 +93,6 @@ TALER_curl_easy_post (struct TALER_CURL_PostContext *ctx, } -/** - * Free the data in @a ctx. - * - * @param[in] ctx a request context (updated) - */ void TALER_curl_easy_post_finished (struct TALER_CURL_PostContext *ctx) { diff --git a/src/lib/exchange_api_common.c b/src/lib/exchange_api_common.c index 4f6588a20..98a6ab20f 100644 --- a/src/lib/exchange_api_common.c +++ b/src/lib/exchange_api_common.c @@ -386,12 +386,6 @@ TALER_EXCHANGE_parse_reserve_history ( } -/** - * Free memory (potentially) allocated by #TALER_EXCHANGE_parse_reserve_history(). - * - * @param rhistory result to free - * @param len number of entries in @a rhistory - */ void TALER_EXCHANGE_free_reserve_history ( struct TALER_EXCHANGE_ReserveHistory *rhistory, @@ -416,17 +410,6 @@ TALER_EXCHANGE_free_reserve_history ( } -/** - * Verify a coins transaction history as returned by the exchange. - * - * @param dk fee structure for the coin, NULL to skip verifying fees - * @param currency expected currency for the coin - * @param coin_pub public key of the coin - * @param history history of the coin in json encoding - * @param[out] h_denom_pub set to the hash of the coin's denomination (if available) - * @param[out] total how much of the coin has been spent according to @a history - * @return #GNUNET_OK if @a history is valid, #GNUNET_SYSERR if not - */ enum GNUNET_GenericReturnValue TALER_EXCHANGE_verify_coin_history ( const struct TALER_EXCHANGE_DenomPublicKey *dk, diff --git a/src/lib/exchange_api_curl_defaults.c b/src/lib/exchange_api_curl_defaults.c index 82d3ace13..6d6016953 100644 --- a/src/lib/exchange_api_curl_defaults.c +++ b/src/lib/exchange_api_curl_defaults.c @@ -1,6 +1,6 @@ /* This file is part of TALER - Copyright (C) 2014-2018 Taler Systems SA + Copyright (C) 2014-2018, 2021 Taler Systems SA TALER is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -23,12 +23,6 @@ #include "exchange_api_curl_defaults.h" -/** - * Get a curl handle with the right defaults for the exchange lib. In the - * future, we might manage a pool of connections here. - * - * @param url URL to query - */ CURL * TALER_EXCHANGE_curl_easy_get_ (const char *url) { @@ -45,6 +39,12 @@ TALER_EXCHANGE_curl_easy_get_ (const char *url) curl_easy_setopt (eh, CURLOPT_FOLLOWLOCATION, 1L)); + /* Enable compression (using whatever curl likes), see + https://curl.se/libcurl/c/CURLOPT_ACCEPT_ENCODING.html */ + GNUNET_break (CURLE_OK == + curl_easy_setopt (eh, + CURLOPT_ACCEPT_ENCODING, + "")); /* limit MAXREDIRS to 5 as a simple security measure against a potential infinite loop caused by a malicious target */ GNUNET_assert (CURLE_OK == diff --git a/src/lib/exchange_api_handle.c b/src/lib/exchange_api_handle.c index 49e39a0e9..5898a0df4 100644 --- a/src/lib/exchange_api_handle.c +++ b/src/lib/exchange_api_handle.c @@ -166,13 +166,6 @@ struct KeysRequest }; -/** - * Signature of functions called with the result from our call to the - * auditor's /deposit-confirmation handler. - * - * @param cls closure of type `struct TEAH_AuditorInteractionEntry *` - * @param hr HTTP response - */ void TEAH_acc_confirmation_cb (void *cls, const struct TALER_AUDITOR_HttpResponse *hr) @@ -195,15 +188,6 @@ TEAH_acc_confirmation_cb (void *cls, } -/** - * Iterate over all available auditors for @a h, calling - * @a ac and giving it a chance to start a deposit - * confirmation interaction. - * - * @param h exchange to go over auditors for - * @param ac function to call per auditor - * @param ac_cls closure for @a ac - */ void TEAH_get_auditors_for_dc (struct TALER_EXCHANGE_Handle *h, TEAH_AuditorCallback ac, @@ -1054,12 +1038,6 @@ static void request_keys (void *cls); -/** - * Let the user set the last valid denomination time manually. - * - * @param exchange the exchange handle. - * @param last_denom_new new last denomination time. - */ void TALER_EXCHANGE_set_last_denom (struct TALER_EXCHANGE_Handle *exchange, struct GNUNET_TIME_Absolute last_denom_new) @@ -1071,14 +1049,6 @@ TALER_EXCHANGE_set_last_denom (struct TALER_EXCHANGE_Handle *exchange, } -/** - * Check if our current response for /keys is valid, and if - * not trigger download. - * - * @param exchange exchange to check keys for - * @param flags options controlling when to download what - * @return until when the response is current, 0 if we are re-downloading - */ struct GNUNET_TIME_Absolute TALER_EXCHANGE_check_keys_current (struct TALER_EXCHANGE_Handle *exchange, enum TALER_EXCHANGE_CheckKeysFlags flags) @@ -1330,12 +1300,6 @@ keys_completed_cb (void *cls, /* ********************* library internal API ********* */ -/** - * Get the context of a exchange. - * - * @param h the exchange handle to query - * @return ctx context to execute jobs in - */ struct GNUNET_CURL_Context * TEAH_handle_to_context (struct TALER_EXCHANGE_Handle *h) { @@ -1343,12 +1307,6 @@ TEAH_handle_to_context (struct TALER_EXCHANGE_Handle *h) } -/** - * Check if the handle is ready to process requests. - * - * @param h the exchange handle to query - * @return #GNUNET_YES if we are ready, #GNUNET_NO if not - */ enum GNUNET_GenericReturnValue TEAH_handle_is_ready (struct TALER_EXCHANGE_Handle *h) { @@ -1356,13 +1314,6 @@ TEAH_handle_is_ready (struct TALER_EXCHANGE_Handle *h) } -/** - * Obtain the URL to use for an API request. - * - * @param h handle for the exchange - * @param path Taler API path (i.e. "/reserve/withdraw") - * @return the full URL to use with cURL - */ char * TEAH_path_to_url (struct TALER_EXCHANGE_Handle *h, const char *path) @@ -1611,17 +1562,6 @@ deserialize_data (struct TALER_EXCHANGE_Handle *exchange, } -/** - * Serialize the latest key data from @a - * exchange to be persisted on disk (to be used with - * #TALER_EXCHANGE_OPTION_DATA to more efficiently recover - * the state). - * - * @param exchange which exchange's key and wire data should be - * serialized - * @return NULL on error (i.e. no current data available); - * otherwise JSON object owned by the caller - */ json_t * TALER_EXCHANGE_serialize_data (struct TALER_EXCHANGE_Handle *exchange) { @@ -1790,24 +1730,6 @@ TALER_EXCHANGE_serialize_data (struct TALER_EXCHANGE_Handle *exchange) } -/** - * Initialise a connection to the exchange. Will connect to the - * exchange and obtain information about the exchange's master - * public key and the exchange's auditor. - * The respective information will be passed to the @a cert_cb - * once available, and all future interactions with the exchange - * will be checked to be signed (where appropriate) by the - * respective master key. - * - * @param ctx the context - * @param url HTTP base URL for the exchange - * @param cert_cb function to call with the exchange's - * certification information - * @param cert_cb_cls closure for @a cert_cb - * @param ... list of additional arguments, - * terminated by #TALER_EXCHANGE_OPTION_END. - * @return the exchange handle; NULL upon error - */ struct TALER_EXCHANGE_Handle * TALER_EXCHANGE_connect ( struct GNUNET_CURL_Context *ctx, @@ -1825,7 +1747,7 @@ TALER_EXCHANGE_connect ( /* Disable 100 continue processing */ GNUNET_break (GNUNET_OK == GNUNET_CURL_append_header (ctx, - "Expect:")); + MHD_HTTP_HEADER_EXPECT ":")); exchange = GNUNET_new (struct TALER_EXCHANGE_Handle); exchange->ctx = ctx; exchange->url = GNUNET_strdup (url); @@ -1968,11 +1890,6 @@ request_keys (void *cls) } -/** - * Disconnect from the exchange - * - * @param exchange the exchange handle - */ void TALER_EXCHANGE_disconnect (struct TALER_EXCHANGE_Handle *exchange) { @@ -2025,14 +1942,6 @@ TALER_EXCHANGE_disconnect (struct TALER_EXCHANGE_Handle *exchange) } -/** - * Test if the given @a pub is a the current signing key from the exchange - * according to @a keys. - * - * @param keys the exchange's key set - * @param pub claimed current online signing key for the exchange - * @return #GNUNET_OK if @a pub is (according to /keys) a current signing key - */ enum GNUNET_GenericReturnValue TALER_EXCHANGE_test_signing_key (const struct TALER_EXCHANGE_Keys *keys, const struct TALER_ExchangePublicKeyP *pub) @@ -2058,12 +1967,6 @@ TALER_EXCHANGE_test_signing_key (const struct TALER_EXCHANGE_Keys *keys, } -/** - * Get exchange's base URL. - * - * @param exchange exchange handle. - * @return the base URL from the handle. - */ const char * TALER_EXCHANGE_get_base_url (const struct TALER_EXCHANGE_Handle *exchange) { @@ -2071,14 +1974,6 @@ TALER_EXCHANGE_get_base_url (const struct TALER_EXCHANGE_Handle *exchange) } -/** - * Obtain the denomination key details from the exchange. - * - * @param keys the exchange's key set - * @param pk public key of the denomination to lookup - * @return details about the given denomination key, NULL if the key is - * not found - */ const struct TALER_EXCHANGE_DenomPublicKey * TALER_EXCHANGE_get_denomination_key ( const struct TALER_EXCHANGE_Keys *keys, @@ -2093,12 +1988,6 @@ TALER_EXCHANGE_get_denomination_key ( } -/** - * Create a copy of a denomination public key. - * - * @param key key to copy - * @returns a copy, must be freed with #TALER_EXCHANGE_destroy_denomination_key - */ struct TALER_EXCHANGE_DenomPublicKey * TALER_EXCHANGE_copy_denomination_key ( const struct TALER_EXCHANGE_DenomPublicKey *key) @@ -2113,12 +2002,6 @@ TALER_EXCHANGE_copy_denomination_key ( } -/** - * Destroy a denomination public key. - * Should only be called with keys created by #TALER_EXCHANGE_copy_denomination_key. - * - * @param key key to destroy. - */ void TALER_EXCHANGE_destroy_denomination_key ( struct TALER_EXCHANGE_DenomPublicKey *key) @@ -2128,13 +2011,6 @@ TALER_EXCHANGE_destroy_denomination_key ( } -/** - * Obtain the denomination key details from the exchange. - * - * @param keys the exchange's key set - * @param hc hash of the public key of the denomination to lookup - * @return details about the given denomination key - */ const struct TALER_EXCHANGE_DenomPublicKey * TALER_EXCHANGE_get_denomination_key_by_hash ( const struct TALER_EXCHANGE_Keys *keys, @@ -2148,12 +2024,6 @@ TALER_EXCHANGE_get_denomination_key_by_hash ( } -/** - * Obtain the keys from the exchange. - * - * @param exchange the exchange handle - * @return the exchange's key set - */ const struct TALER_EXCHANGE_Keys * TALER_EXCHANGE_get_keys (struct TALER_EXCHANGE_Handle *exchange) { @@ -2163,13 +2033,6 @@ TALER_EXCHANGE_get_keys (struct TALER_EXCHANGE_Handle *exchange) } -/** - * Obtain the keys from the exchange in the - * raw JSON format - * - * @param exchange the exchange handle - * @return the exchange's keys in raw JSON - */ json_t * TALER_EXCHANGE_get_keys_raw (struct TALER_EXCHANGE_Handle *exchange) { diff --git a/src/lib/exchange_api_link.c b/src/lib/exchange_api_link.c index a9dc8852f..55d3bdb66 100644 --- a/src/lib/exchange_api_link.c +++ b/src/lib/exchange_api_link.c @@ -418,19 +418,6 @@ handle_link_finished (void *cls, } -/** - * Submit a link request to the exchange and get the exchange's response. - * - * This API is typically not used by anyone, it is more a threat against those - * trying to receive a funds transfer by abusing the refresh protocol. - * - * @param exchange the exchange handle; the exchange must be ready to operate - * @param coin_priv private key to request link data for - * @param link_cb the callback to call with the useful result of the - * refresh operation the @a coin_priv was involved in (if any) - * @param link_cb_cls closure for @a link_cb - * @return a handle for this request - */ struct TALER_EXCHANGE_LinkHandle * TALER_EXCHANGE_link (struct TALER_EXCHANGE_Handle *exchange, const struct TALER_CoinSpendPrivateKeyP *coin_priv, @@ -496,12 +483,6 @@ TALER_EXCHANGE_link (struct TALER_EXCHANGE_Handle *exchange, } -/** - * Cancel a link request. This function cannot be used - * on a request handle if the callback was already invoked. - * - * @param lh the link handle - */ void TALER_EXCHANGE_link_cancel (struct TALER_EXCHANGE_LinkHandle *lh) { From 57f493345149437bc964e9695558646f654c9557 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 11 Dec 2021 21:53:30 +0100 Subject: [PATCH 37/37] fix uncrustify --- contrib/uncrustify.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/uncrustify.cfg b/contrib/uncrustify.cfg index 8c9df2c43..af2d8e69c 100644 --- a/contrib/uncrustify.cfg +++ b/contrib/uncrustify.cfg @@ -28,7 +28,7 @@ ls_code_width=true pos_arith=lead # Fully parenthesize boolean exprs -mod_full_paren_if_bool=true +mod_full_paren_if_bool=false # Braces should be on their own line nl_fdef_brace=add