diff --git a/src/benchmark/.gitignore b/src/benchmark/.gitignore index 9757b376b..45e150087 100644 --- a/src/benchmark/.gitignore +++ b/src/benchmark/.gitignore @@ -1 +1,2 @@ taler-bank-benchmark +taler-aggregator-benchmark diff --git a/src/benchmark/Makefile.am b/src/benchmark/Makefile.am index 1ead23df4..2965d62dc 100644 --- a/src/benchmark/Makefile.am +++ b/src/benchmark/Makefile.am @@ -11,9 +11,21 @@ if USE_COVERAGE endif bin_PROGRAMS = \ + taler-aggregator-benchmark \ taler-bank-benchmark \ taler-exchange-benchmark +taler_aggregator_benchmark_SOURCES = \ + taler-aggregator-benchmark.c +taler_aggregator_benchmark_LDADD = \ + $(LIBGCRYPT_LIBS) \ + $(top_builddir)/src/exchangedb/libtalerexchangedb.la \ + $(top_builddir)/src/util/libtalerutil.la \ + -lgnunetjson \ + -ljansson \ + -lgnunetutil \ + $(XLIB) + taler_bank_benchmark_SOURCES = \ taler-bank-benchmark.c taler_bank_benchmark_LDADD = \ diff --git a/src/benchmark/benchmark.conf b/src/benchmark/benchmark.conf index 3a11b73ea..5716770c3 100644 --- a/src/benchmark/benchmark.conf +++ b/src/benchmark/benchmark.conf @@ -68,6 +68,11 @@ username = Exchange password = x +[exchange-account-aggregator] +# What is the payto://-URL of the exchange (to generate wire response) +PAYTO_URI = "payto://aggregator-benchmark/exchangeacc" +enable_debit = YES + # Sections starting with "coin_" specify which denominations diff --git a/src/benchmark/taler-aggregator-benchmark.c b/src/benchmark/taler-aggregator-benchmark.c new file mode 100644 index 000000000..67bef7052 --- /dev/null +++ b/src/benchmark/taler-aggregator-benchmark.c @@ -0,0 +1,593 @@ +/* + This file is part of TALER + (C) 2021 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as + published by the Free Software Foundation; either version 3, or + (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TALER; see the file COPYING. If not, + see +*/ +/** + * @file benchmark/taler-aggregator-benchmark.c + * @brief Setup exchange database suitable for aggregator benchmarking + * @author Christian Grothoff + */ +#include "platform.h" +#include +#include +#include +#include "taler_util.h" +#include "taler_signatures.h" +#include "taler_exchangedb_lib.h" +#include "taler_error_codes.h" + + +/** + * Exit code. + */ +static int global_ret; + +/** + * How many deposits we want to create per merchant. + */ +static unsigned int howmany_deposits = 1; + +/** + * How many merchants do we want to setup. + */ +static unsigned int howmany_merchants = 1; + +/** + * Probability of a refund, as in $NUMBER:100. + * Use 0 for no refunds. + */ +static unsigned int refund_rate = 0; + +/** + * Currency used. + */ +static char *currency; + +/** + * Merchant JSON wire details. + */ +static json_t *json_wire; + +/** + * Configuration. + */ +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +/** + * Database plugin. + */ +static struct TALER_EXCHANGEDB_Plugin *plugin; + +/** + * Main task doing the work(). + */ +static struct GNUNET_SCHEDULER_Task *task; + +/** + * Hash of the denomination. + */ +static struct GNUNET_HashCode h_denom_pub; + +/** + * "signature" to use for the coin(s). + */ +static struct TALER_DenominationSignature denom_sig; + +/** + * Time range when deposits start. + */ +static struct GNUNET_TIME_Absolute start; + +/** + * Time range when deposits end. + */ +static struct GNUNET_TIME_Absolute end; + + +/** + * Throw a weighted coin with @a probability. + * + * @return #GNUNET_OK with @a probability, + * #GNUNET_NO with 1 - @a probability + */ +static unsigned int +eval_probability (float probability) +{ + uint64_t random; + float random_01; + + random = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, + UINT64_MAX); + random_01 = (double) random / (double) UINT64_MAX; + return (random_01 <= probability) ? GNUNET_OK : GNUNET_NO; +} + + +/** + * Randomize data at pointer @a x + * + * @param x pointer to data to randomize + */ +#define RANDOMIZE(x) \ + GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE, x, sizeof (*x)) + + +/** + * Initialize @a out with an amount given by @a val and + * @a frac using the main "currency". + * + * @param val value to set + * @param frac fraction to set + * @param[out] out where to write the amount + */ +static void +make_amount (unsigned int val, + unsigned int frac, + struct TALER_Amount *out) +{ + memset (out, + 0, + sizeof (struct TALER_Amount)); + out->value = val; + out->fraction = frac; + strcpy (out->currency, + currency); +} + + +/** + * Initialize @a out with an amount given by @a val and + * @a frac using the main "currency". + * + * @param val value to set + * @param frac fraction to set + * @param[out] out where to write the amount + */ +static void +make_amountN (unsigned int val, + unsigned int frac, + struct TALER_AmountNBO *out) +{ + struct TALER_Amount in; + + make_amount (val, + frac, + &in); + TALER_amount_hton (out, + &in); +} + + +/** + * Create random-ish timestamp. + * + * @return time stamp between start and end + */ +static struct GNUNET_TIME_Absolute +random_time (void) +{ + uint64_t delta; + struct GNUNET_TIME_Absolute ret; + + delta = end.abs_value_us - start.abs_value_us; + delta = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_NONCE, + delta); + ret.abs_value_us = start.abs_value_us + delta; + (void) GNUNET_TIME_round_abs (&ret); + return ret; +} + + +/** + * Function run on shutdown. + * + * @param cls unused + */ +static void +do_shutdown (void *cls) +{ + (void) cls; + if (NULL != plugin) + { + TALER_EXCHANGEDB_plugin_unload (plugin); + plugin = NULL; + } + if (NULL != task) + { + GNUNET_SCHEDULER_cancel (task); + task = NULL; + } + if (NULL !=denom_sig.rsa_signature) + { + GNUNET_CRYPTO_rsa_signature_free (denom_sig.rsa_signature); + denom_sig.rsa_signature = NULL; + } + if (NULL != json_wire) + { + json_decref (json_wire); + json_wire = NULL; + } +} + + +struct Merchant +{ + + /** + * Public key of the merchant. Enables later identification + * of the merchant in case of a need to rollback transactions. + */ + struct TALER_MerchantPublicKeyP merchant_pub; + + /** + * Hash of the (canonical) representation of @e wire, used + * to check the signature on the request. Generated by + * the exchange from the detailed wire data provided by the + * merchant. + */ + struct GNUNET_HashCode h_wire; + +}; + +struct Deposit +{ + + /** + * Information about the coin that is being deposited. + */ + struct TALER_CoinPublicInfo coin; + + /** + * Hash over the proposal data between merchant and customer + * (remains unknown to the Exchange). + */ + struct GNUNET_HashCode h_contract_terms; + +}; + + +/** + * Add a refund from @a m for @a d. + * + * @param m merchant granting the refund + * @param d deposit being refunded + * @return true on success + */ +static bool +add_refund (const struct Merchant *m, + const struct Deposit *d) +{ + struct TALER_EXCHANGEDB_Refund r; + + r.coin = d->coin; + r.details.merchant_pub = m->merchant_pub; + RANDOMIZE (&r.details.merchant_sig); + r.details.h_contract_terms = d->h_contract_terms; + r.details.rtransaction_id = 42; + make_amount (0, 9999, &r.details.refund_amount); + make_amount (0, 5, &r.details.refund_fee); + if (0 <= + plugin->insert_refund (plugin->cls, + &r)) + { + GNUNET_break (0); + global_ret = EXIT_FAILURE; + GNUNET_SCHEDULER_shutdown (); + return false; + } + return true; +} + + +/** + * Add a (random-ish) deposit for merchant @a m. + * + * @param m merchant to receive the deposit + * @return true on success + */ +static bool +add_deposit (const struct Merchant *m) +{ + struct Deposit d; + struct TALER_EXCHANGEDB_Deposit deposit; + + RANDOMIZE (&d.coin.coin_pub); + d.coin.denom_pub_hash = h_denom_pub; + d.coin.denom_sig = denom_sig; + RANDOMIZE (&d.h_contract_terms); + + if (0 >= + plugin->ensure_coin_known (plugin->cls, + &d.coin)) + { + GNUNET_break (0); + global_ret = EXIT_FAILURE; + GNUNET_SCHEDULER_shutdown (); + return false; + } + deposit.coin = d.coin; + RANDOMIZE (&deposit.csig); + deposit.merchant_pub = m->merchant_pub; + deposit.h_contract_terms = d.h_contract_terms; + deposit.h_wire = m->h_wire; + deposit.receiver_wire_account + = json_wire; + deposit.timestamp = random_time (); + deposit.refund_deadline = random_time (); + deposit.wire_deadline = random_time (); + make_amount (0, 99999, &deposit.amount_with_fee); + make_amount (0, 5, &deposit.deposit_fee); + if (0 >= + plugin->insert_deposit (plugin->cls, + random_time (), + &deposit)) + { + GNUNET_break (0); + global_ret = EXIT_FAILURE; + GNUNET_SCHEDULER_shutdown (); + return false; + } + if (GNUNET_YES == + eval_probability (((float) refund_rate) / 100.0)) + return add_refund (m, + &d); + return true; +} + + +/** + * Function to do the work. + * + * @param cls unused + */ +static void +work (void *cls) +{ + struct Merchant m; + + (void) cls; + task = NULL; + RANDOMIZE (&m); + if (GNUNET_OK != + plugin->start (plugin->cls, + "aggregator-benchmark-fill")) + { + GNUNET_break (0); + global_ret = EXIT_FAILURE; + GNUNET_SCHEDULER_shutdown (); + return; + } + for (unsigned int i = 0; icommit (plugin->cls)) + { + if (0 == --howmany_merchants) + { + GNUNET_SCHEDULER_shutdown (); + return; + } + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Failed to commit, will try again\n"); + } + task = GNUNET_SCHEDULER_add_now (&work, + NULL); +} + + +/** + * Actual execution. + * + * @param cls unused + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param cfg configuration + */ +static void +run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + struct TALER_EXCHANGEDB_DenominationKeyInformationP issue; + + (void) cls; + /* make sure everything 'ends' before the current time, + so that the aggregator will process everything without + need for time-travel */ + end = GNUNET_TIME_absolute_get (); + (void) GNUNET_TIME_round_abs (&end); + start = GNUNET_TIME_absolute_subtract (end, + GNUNET_TIME_UNIT_MONTHS); + (void) GNUNET_TIME_round_abs (&start); + cfg = c; + if (GNUNET_OK != + TALER_config_get_currency (cfg, + ¤cy)) + { + global_ret = EXIT_NOTCONFIGURED; + return; + } + plugin = TALER_EXCHANGEDB_plugin_load (cfg); + if (NULL == plugin) + { + global_ret = EXIT_NOTCONFIGURED; + return; + } + if (GNUNET_SYSERR == + plugin->preflight (plugin->cls)) + { + global_ret = EXIT_FAILURE; + TALER_EXCHANGEDB_plugin_unload (plugin); + return; + } + GNUNET_SCHEDULER_add_shutdown (&do_shutdown, + NULL); + RANDOMIZE (&issue.signature); + issue.properties.purpose.purpose = htonl ( + TALER_SIGNATURE_MASTER_DENOMINATION_KEY_VALIDITY); + issue.properties.purpose.size = htonl (sizeof (issue.properties)); + RANDOMIZE (&issue.properties.master); + issue.properties.start + = GNUNET_TIME_absolute_hton (start); + issue.properties.expire_withdraw + = GNUNET_TIME_absolute_hton ( + GNUNET_TIME_absolute_add (start, + GNUNET_TIME_UNIT_DAYS)); + issue.properties.expire_deposit + = GNUNET_TIME_absolute_hton (end); + issue.properties.expire_legal + = GNUNET_TIME_absolute_hton ( + GNUNET_TIME_absolute_add (end, + GNUNET_TIME_UNIT_YEARS)); + { + struct GNUNET_CRYPTO_RsaPrivateKey *pk; + struct GNUNET_CRYPTO_RsaPublicKey *pub; + struct GNUNET_HashCode hc; + struct TALER_DenominationPublicKey denom_pub; + + RANDOMIZE (&hc); + pk = GNUNET_CRYPTO_rsa_private_key_create (1024); + pub = GNUNET_CRYPTO_rsa_private_key_get_public (pk); + denom_pub.rsa_public_key = pub; + GNUNET_CRYPTO_rsa_public_key_hash (pub, + &h_denom_pub); + make_amountN (1, 0, &issue.properties.value); + make_amountN (0, 5, &issue.properties.fee_withdraw); + make_amountN (0, 5, &issue.properties.fee_deposit); + make_amountN (0, 5, &issue.properties.fee_refresh); + make_amountN (0, 5, &issue.properties.fee_refund); + issue.properties.denom_hash = h_denom_pub; + if (0 >= + plugin->insert_denomination_info (plugin->cls, + &denom_pub, + &issue)) + { + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + global_ret = EXIT_FAILURE; + return; + } + + denom_sig.rsa_signature + = GNUNET_CRYPTO_rsa_sign_fdh (pk, + &hc); + GNUNET_CRYPTO_rsa_public_key_free (pub); + GNUNET_CRYPTO_rsa_private_key_free (pk); + } + + { + struct TALER_Amount wire_fee; + struct TALER_MasterSignatureP master_sig; + unsigned int year; + struct GNUNET_TIME_Absolute ws; + struct GNUNET_TIME_Absolute we; + + year = GNUNET_TIME_get_current_year (); + for (unsigned int y = year - 1; y + plugin->insert_wire_fee (plugin->cls, + "aggregator-benchmark", + ws, + we, + &wire_fee, + &wire_fee, + &master_sig)) + { + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + global_ret = EXIT_FAILURE; + return; + } + } + } + + json_wire = GNUNET_JSON_PACK ( + GNUNET_JSON_pack_string ("payto_uri", + "payto://aggregator-benchmark/accountfoo"), + GNUNET_JSON_pack_string ("salt", + "thesalty")); + task = GNUNET_SCHEDULER_add_now (&work, + NULL); +} + + +/** + * The main function of the taler-aggregator-benchmark tool. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, non-zero on failure + */ +int +main (int argc, + char *const *argv) +{ + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_uint ('d', + "deposits", + "DN", + "How many deposits we should instantiate per merchant", + &howmany_deposits), + GNUNET_GETOPT_option_uint ('m', + "merchants", + "DM", + "How many merchants should we create", + &howmany_merchants), + GNUNET_GETOPT_option_uint ('r', + "refunds", + "RATE", + "Probability of refund per deposit (0-100)", + &refund_rate), + GNUNET_GETOPT_OPTION_END + }; + enum GNUNET_GenericReturnValue result; + + unsetenv ("XDG_DATA_HOME"); + unsetenv ("XDG_CONFIG_HOME"); + if (0 >= + (result = GNUNET_PROGRAM_run (argc, + argv, + "taler-aggregator-benchmark", + "generate database to benchmark the aggregator", + options, + &run, + NULL))) + { + if (GNUNET_NO == result) + return EXIT_SUCCESS; + return EXIT_INVALIDARGUMENT; + } + return global_ret; +} diff --git a/src/exchange-tools/taler-exchange-dbinit.c b/src/exchange-tools/taler-exchange-dbinit.c index 5e18549e7..b187cff33 100644 --- a/src/exchange-tools/taler-exchange-dbinit.c +++ b/src/exchange-tools/taler-exchange-dbinit.c @@ -89,20 +89,33 @@ run (void *cls, global_ret = EXIT_NOPERMISSION; return; } - if (clear_shards) + if (gc_db || clear_shards) { - if (0 < plugin->delete_revolving_shards (plugin->cls)) + if (GNUNET_OK != + plugin->preflight (plugin->cls)) { fprintf (stderr, - "Clearing revolving shards failed!\n"); + "Failed to prepare database.\n"); + TALER_EXCHANGEDB_plugin_unload (plugin); + global_ret = EXIT_NOPERMISSION; + return; } - } - if (gc_db) - { - if (GNUNET_SYSERR == plugin->gc (plugin->cls)) + if (clear_shards) { - fprintf (stderr, - "Garbage collection failed!\n"); + if (0 > + plugin->delete_revolving_shards (plugin->cls)) + { + fprintf (stderr, + "Clearing revolving shards failed!\n"); + } + } + if (gc_db) + { + if (GNUNET_SYSERR == plugin->gc (plugin->cls)) + { + fprintf (stderr, + "Garbage collection failed!\n"); + } } } TALER_EXCHANGEDB_plugin_unload (plugin); diff --git a/src/exchange/taler-exchange-aggregator.c b/src/exchange/taler-exchange-aggregator.c index 0fc13c145..eb1b548e2 100644 --- a/src/exchange/taler-exchange-aggregator.c +++ b/src/exchange/taler-exchange-aggregator.c @@ -124,7 +124,7 @@ struct Shard uint32_t shard_start; /** - * Exclusive end row of the shard. + * Inclusive end row of the shard. */ uint32_t shard_end; @@ -724,7 +724,7 @@ run_aggregation (void *cls) qs = db_plugin->get_ready_deposit ( db_plugin->cls, s->shard_start, - s->shard_end - 1, /* -1: exclusive->inclusive */ + s->shard_end, &deposit_cb, &au_active); switch (qs) @@ -754,9 +754,12 @@ run_aggregation (void *cls) cleanup_au (&au_active); db_plugin->rollback (db_plugin->cls); GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Completed shard after %s\n", + "Completed shard [%u,%u] after %s with %llu deposits\n", + (unsigned int) s->shard_start, + (unsigned int) s->shard_end, GNUNET_STRINGS_relative_time_to_string (duration, - GNUNET_YES)); + GNUNET_YES), + (unsigned long long) counter); release_shard (s); if (GNUNET_YES == test_mode) { @@ -1035,6 +1038,10 @@ run_shard (void *cls) GNUNET_SCHEDULER_shutdown (); return; } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Starting shard [%u:%u]!\n", + (unsigned int) s->shard_start, + (unsigned int) s->shard_end); task = GNUNET_SCHEDULER_add_now (&run_aggregation, s); } diff --git a/src/exchangedb/plugin_exchangedb_postgres.c b/src/exchangedb/plugin_exchangedb_postgres.c index 70c337c57..961921eaa 100644 --- a/src/exchangedb/plugin_exchangedb_postgres.c +++ b/src/exchangedb/plugin_exchangedb_postgres.c @@ -10488,7 +10488,7 @@ postgres_complete_shard (void *cls, * @param shard_size desired shard size * @param shard_limit exclusive end of the shard range * @param[out] start_row inclusive start row of the shard (returned) - * @param[out] end_row exclusive end row of the shard (returned) + * @param[out] end_row inclusive end row of the shard (returned) * @return transaction status code */ static enum GNUNET_DB_QueryStatus @@ -10517,13 +10517,14 @@ postgres_begin_revolving_shard (void *cls, /* First, find last 'end_row' */ { enum GNUNET_DB_QueryStatus qs; + uint32_t last_end; struct GNUNET_PQ_QueryParam params[] = { GNUNET_PQ_query_param_string (job_name), GNUNET_PQ_query_param_end }; struct GNUNET_PQ_ResultSpec rs[] = { GNUNET_PQ_result_spec_uint32 ("end_row", - start_row), + &last_end), GNUNET_PQ_result_spec_end }; @@ -10541,6 +10542,7 @@ postgres_begin_revolving_shard (void *cls, postgres_rollback (pg); continue; case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + *start_row = 1U + last_end; break; case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: *start_row = 0; /* base-case: no shards yet */ @@ -10562,7 +10564,7 @@ postgres_begin_revolving_shard (void *cls, }; *end_row = GNUNET_MIN (shard_limit, - *start_row + shard_size); + *start_row + shard_size - 1); now = GNUNET_TIME_absolute_get (); GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Trying to claim shard %llu-%llu\n", diff --git a/src/include/taler_exchangedb_plugin.h b/src/include/taler_exchangedb_plugin.h index 2faf331d5..7d6508fc5 100644 --- a/src/include/taler_exchangedb_plugin.h +++ b/src/include/taler_exchangedb_plugin.h @@ -974,7 +974,7 @@ struct TALER_EXCHANGEDB_Deposit struct TALER_MerchantPublicKeyP merchant_pub; /** - * Hash over the proposa data between merchant and customer + * Hash over the proposal data between merchant and customer * (remains unknown to the Exchange). */ struct GNUNET_HashCode h_contract_terms; @@ -3760,7 +3760,7 @@ struct TALER_EXCHANGEDB_Plugin * @param shard_size desired shard size * @param shard_limit exclusive end of the shard range * @param[out] start_row inclusive start row of the shard (returned) - * @param[out] end_row exclusive end row of the shard (returned) + * @param[out] end_row inclusive end row of the shard (returned) * @return transaction status code */ enum GNUNET_DB_QueryStatus @@ -3779,7 +3779,7 @@ struct TALER_EXCHANGEDB_Plugin * @param cls the @e cls of this struct with the plugin-specific state * @param job_name name of the operation to grab a word shard for * @param start_row inclusive start row of the shard - * @param end_row exclusive end row of the shard + * @param end_row inclusive end row of the shard * @return transaction status code */ enum GNUNET_DB_QueryStatus diff --git a/src/json/json_wire.c b/src/json/json_wire.c index 3d7e8a81b..e8620728a 100644 --- a/src/json/json_wire.c +++ b/src/json/json_wire.c @@ -37,9 +37,10 @@ TALER_JSON_merchant_wire_signature_hash (const json_t *wire_s, struct GNUNET_HashCode *hc) { const char *payto_uri; - const char *salt; /* Current merchant backend will always make the salt - a `struct GNUNET_HashCode`, but *we* do not insist - on that. */ + const char *salt; + /* Current merchant backend will always make the salt + a `struct GNUNET_HashCode`, but *we* do not insist + on that. */ struct GNUNET_JSON_Specification spec[] = { GNUNET_JSON_spec_string ("payto_uri", &payto_uri), GNUNET_JSON_spec_string ("salt", &salt),