taler/exchange
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

first draft for an aggregator benchmark, plus fixing inclusive/exclusive sharding range issues

Browse Source
This commit is contained in:
Christian Grothoff 2021-09-03 21:30:14 +02:00
parent 5149af9314
commit 1c1d28d3f5
No known key found for this signature in database
GPG Key ID: 939E6BE1E29FC3CC
9 changed files with 656 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/benchmark/.gitignore vendored
View File

@ -1 +1,2 @@
taler-bank-benchmark
taler-aggregator-benchmark

12
src/benchmark/Makefile.am
View File

@ -11,9 +11,21 @@ if USE_COVERAGE
endif
bin_PROGRAMS = \
taler-aggregator-benchmark \
taler-bank-benchmark \
taler-exchange-benchmark
taler_aggregator_benchmark_SOURCES = \
taler-aggregator-benchmark.c
taler_aggregator_benchmark_LDADD = \
$(LIBGCRYPT_LIBS) \
$(top_builddir)/src/exchangedb/libtalerexchangedb.la \
$(top_builddir)/src/util/libtalerutil.la \
-lgnunetjson \
-ljansson \
-lgnunetutil \
$(XLIB)
taler_bank_benchmark_SOURCES = \
taler-bank-benchmark.c
taler_bank_benchmark_LDADD = \

5
src/benchmark/benchmark.conf
View File

@ -68,6 +68,11 @@ username = Exchange
password = x
[exchange-account-aggregator]
# What is the payto://-URL of the exchange (to generate wire response)
PAYTO_URI = "payto://aggregator-benchmark/exchangeacc"
enable_debit = YES
# Sections starting with "coin_" specify which denominations

593
src/benchmark/taler-aggregator-benchmark.c Normal file
View File

@ -0,0 +1,593 @@
/*
This file is part of TALER
(C) 2021 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License as
published by the Free Software Foundation; either version 3, or
(at your option) any later version.
TALER is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with TALER; see the file COPYING. If not,
see <http://www.gnu.org/licenses/>
*/
/**
* @file benchmark/taler-aggregator-benchmark.c
* @brief Setup exchange database suitable for aggregator benchmarking
* @author Christian Grothoff
*/
#include "platform.h"
#include <jansson.h>
#include <gnunet/gnunet_util_lib.h>
#include <gnunet/gnunet_json_lib.h>
#include "taler_util.h"
#include "taler_signatures.h"
#include "taler_exchangedb_lib.h"
#include "taler_error_codes.h"
/**
* Exit code.
*/
static int global_ret;
/**
* How many deposits we want to create per merchant.
*/
static unsigned int howmany_deposits = 1;
/**
* How many merchants do we want to setup.
*/
static unsigned int howmany_merchants = 1;
/**
* Probability of a refund, as in $NUMBER:100.
* Use 0 for no refunds.
*/
static unsigned int refund_rate = 0;
/**
* Currency used.
*/
static char *currency;
/**
* Merchant JSON wire details.
*/
static json_t *json_wire;
/**
* Configuration.
*/
static const struct GNUNET_CONFIGURATION_Handle *cfg;
/**
* Database plugin.
*/
static struct TALER_EXCHANGEDB_Plugin *plugin;
/**
* Main task doing the work().
*/
static struct GNUNET_SCHEDULER_Task *task;
/**
* Hash of the denomination.
*/
static struct GNUNET_HashCode h_denom_pub;
/**
* "signature" to use for the coin(s).
*/
static struct TALER_DenominationSignature denom_sig;
/**
* Time range when deposits start.
*/
static struct GNUNET_TIME_Absolute start;
/**
* Time range when deposits end.
*/
static struct GNUNET_TIME_Absolute end;
/**
* Throw a weighted coin with @a probability.
*
* @return #GNUNET_OK with @a probability,
* #GNUNET_NO with 1 - @a probability
*/
static unsigned int
eval_probability (float probability)
{
uint64_t random;
float random_01;
random = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
UINT64_MAX);
random_01 = (double) random / (double) UINT64_MAX;
return (random_01 <= probability) ? GNUNET_OK : GNUNET_NO;
}
/**
* Randomize data at pointer @a x
*
* @param x pointer to data to randomize
*/
#define RANDOMIZE(x) \
GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE, x, sizeof (*x))
/**
* Initialize @a out with an amount given by @a val and
* @a frac using the main "currency".
*
* @param val value to set
* @param frac fraction to set
* @param[out] out where to write the amount
*/
static void
make_amount (unsigned int val,
unsigned int frac,
struct TALER_Amount *out)
{
memset (out,
0,
sizeof (struct TALER_Amount));
out->value = val;
out->fraction = frac;
strcpy (out->currency,
currency);
}
/**
* Initialize @a out with an amount given by @a val and
* @a frac using the main "currency".
*
* @param val value to set
* @param frac fraction to set
* @param[out] out where to write the amount
*/
static void
make_amountN (unsigned int val,
unsigned int frac,
struct TALER_AmountNBO *out)
{
struct TALER_Amount in;
make_amount (val,
frac,
&in);
TALER_amount_hton (out,
&in);
}
/**
* Create random-ish timestamp.
*
* @return time stamp between start and end
*/
static struct GNUNET_TIME_Absolute
random_time (void)
{
uint64_t delta;
struct GNUNET_TIME_Absolute ret;
delta = end.abs_value_us - start.abs_value_us;
delta = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_NONCE,
delta);
ret.abs_value_us = start.abs_value_us + delta;
(void) GNUNET_TIME_round_abs (&ret);
return ret;
}
/**
* Function run on shutdown.
*
* @param cls unused
*/
static void
do_shutdown (void *cls)
{
(void) cls;
if (NULL != plugin)
{
TALER_EXCHANGEDB_plugin_unload (plugin);
plugin = NULL;
}
if (NULL != task)
{
GNUNET_SCHEDULER_cancel (task);
task = NULL;
}
if (NULL !=denom_sig.rsa_signature)
{
GNUNET_CRYPTO_rsa_signature_free (denom_sig.rsa_signature);
denom_sig.rsa_signature = NULL;
}
if (NULL != json_wire)
{
json_decref (json_wire);
json_wire = NULL;
}
}
struct Merchant
{
/**
* Public key of the merchant. Enables later identification
* of the merchant in case of a need to rollback transactions.
*/
struct TALER_MerchantPublicKeyP merchant_pub;
/**
* Hash of the (canonical) representation of @e wire, used
* to check the signature on the request. Generated by
* the exchange from the detailed wire data provided by the
* merchant.
*/
struct GNUNET_HashCode h_wire;
};
struct Deposit
{
/**
* Information about the coin that is being deposited.
*/
struct TALER_CoinPublicInfo coin;
/**
* Hash over the proposal data between merchant and customer
* (remains unknown to the Exchange).
*/
struct GNUNET_HashCode h_contract_terms;
};
/**
* Add a refund from @a m for @a d.
*
* @param m merchant granting the refund
* @param d deposit being refunded
* @return true on success
*/
static bool
add_refund (const struct Merchant *m,
const struct Deposit *d)
{
struct TALER_EXCHANGEDB_Refund r;
r.coin = d->coin;
r.details.merchant_pub = m->merchant_pub;
RANDOMIZE (&r.details.merchant_sig);
r.details.h_contract_terms = d->h_contract_terms;
r.details.rtransaction_id = 42;
make_amount (0, 9999, &r.details.refund_amount);
make_amount (0, 5, &r.details.refund_fee);
if (0 <=
plugin->insert_refund (plugin->cls,
&r))
{
GNUNET_break (0);
global_ret = EXIT_FAILURE;
GNUNET_SCHEDULER_shutdown ();
return false;
}
return true;
}
/**
* Add a (random-ish) deposit for merchant @a m.
*
* @param m merchant to receive the deposit
* @return true on success
*/
static bool
add_deposit (const struct Merchant *m)
{
struct Deposit d;
struct TALER_EXCHANGEDB_Deposit deposit;
RANDOMIZE (&d.coin.coin_pub);
d.coin.denom_pub_hash = h_denom_pub;
d.coin.denom_sig = denom_sig;
RANDOMIZE (&d.h_contract_terms);
if (0 >=
plugin->ensure_coin_known (plugin->cls,
&d.coin))
{
GNUNET_break (0);
global_ret = EXIT_FAILURE;
GNUNET_SCHEDULER_shutdown ();
return false;
}
deposit.coin = d.coin;
RANDOMIZE (&deposit.csig);
deposit.merchant_pub = m->merchant_pub;
deposit.h_contract_terms = d.h_contract_terms;
deposit.h_wire = m->h_wire;
deposit.receiver_wire_account
= json_wire;
deposit.timestamp = random_time ();
deposit.refund_deadline = random_time ();
deposit.wire_deadline = random_time ();
make_amount (0, 99999, &deposit.amount_with_fee);
make_amount (0, 5, &deposit.deposit_fee);
if (0 >=
plugin->insert_deposit (plugin->cls,
random_time (),
&deposit))
{
GNUNET_break (0);
global_ret = EXIT_FAILURE;
GNUNET_SCHEDULER_shutdown ();
return false;
}
if (GNUNET_YES ==
eval_probability (((float) refund_rate) / 100.0))
return add_refund (m,
&d);
return true;
}
/**
* Function to do the work.
*
* @param cls unused
*/
static void
work (void *cls)
{
struct Merchant m;
(void) cls;
task = NULL;
RANDOMIZE (&m);
if (GNUNET_OK !=
plugin->start (plugin->cls,
"aggregator-benchmark-fill"))
{
GNUNET_break (0);
global_ret = EXIT_FAILURE;
GNUNET_SCHEDULER_shutdown ();
return;
}
for (unsigned int i = 0; i<howmany_deposits; i++)
{
if (! add_deposit (&m))
{
global_ret = EXIT_FAILURE;
GNUNET_SCHEDULER_shutdown ();
return;
}
}
if (0 <=
plugin->commit (plugin->cls))
{
if (0 == --howmany_merchants)
{
GNUNET_SCHEDULER_shutdown ();
return;
}
}
else
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Failed to commit, will try again\n");
}
task = GNUNET_SCHEDULER_add_now (&work,
NULL);
}
/**
* Actual execution.
*
* @param cls unused
* @param args remaining command-line arguments
* @param cfgfile name of the configuration file used (for saving, can be NULL!)
* @param cfg configuration
*/
static void
run (void *cls,
char *const *args,
const char *cfgfile,
const struct GNUNET_CONFIGURATION_Handle *c)
{
struct TALER_EXCHANGEDB_DenominationKeyInformationP issue;
(void) cls;
/* make sure everything 'ends' before the current time,
so that the aggregator will process everything without
need for time-travel */
end = GNUNET_TIME_absolute_get ();
(void) GNUNET_TIME_round_abs (&end);
start = GNUNET_TIME_absolute_subtract (end,
GNUNET_TIME_UNIT_MONTHS);
(void) GNUNET_TIME_round_abs (&start);
cfg = c;
if (GNUNET_OK !=
TALER_config_get_currency (cfg,
&currency))
{
global_ret = EXIT_NOTCONFIGURED;
return;
}
plugin = TALER_EXCHANGEDB_plugin_load (cfg);
if (NULL == plugin)
{
global_ret = EXIT_NOTCONFIGURED;
return;
}
if (GNUNET_SYSERR ==
plugin->preflight (plugin->cls))
{
global_ret = EXIT_FAILURE;
TALER_EXCHANGEDB_plugin_unload (plugin);
return;
}
GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
NULL);
RANDOMIZE (&issue.signature);
issue.properties.purpose.purpose = htonl (
TALER_SIGNATURE_MASTER_DENOMINATION_KEY_VALIDITY);
issue.properties.purpose.size = htonl (sizeof (issue.properties));
RANDOMIZE (&issue.properties.master);
issue.properties.start
= GNUNET_TIME_absolute_hton (start);
issue.properties.expire_withdraw
= GNUNET_TIME_absolute_hton (
GNUNET_TIME_absolute_add (start,
GNUNET_TIME_UNIT_DAYS));
issue.properties.expire_deposit
= GNUNET_TIME_absolute_hton (end);
issue.properties.expire_legal
= GNUNET_TIME_absolute_hton (
GNUNET_TIME_absolute_add (end,
GNUNET_TIME_UNIT_YEARS));
{
struct GNUNET_CRYPTO_RsaPrivateKey *pk;
struct GNUNET_CRYPTO_RsaPublicKey *pub;
struct GNUNET_HashCode hc;
struct TALER_DenominationPublicKey denom_pub;
RANDOMIZE (&hc);
pk = GNUNET_CRYPTO_rsa_private_key_create (1024);
pub = GNUNET_CRYPTO_rsa_private_key_get_public (pk);
denom_pub.rsa_public_key = pub;
GNUNET_CRYPTO_rsa_public_key_hash (pub,
&h_denom_pub);
make_amountN (1, 0, &issue.properties.value);
make_amountN (0, 5, &issue.properties.fee_withdraw);
make_amountN (0, 5, &issue.properties.fee_deposit);
make_amountN (0, 5, &issue.properties.fee_refresh);
make_amountN (0, 5, &issue.properties.fee_refund);
issue.properties.denom_hash = h_denom_pub;
if (0 >=
plugin->insert_denomination_info (plugin->cls,
&denom_pub,
&issue))
{
GNUNET_break (0);
GNUNET_SCHEDULER_shutdown ();
global_ret = EXIT_FAILURE;
return;
}
denom_sig.rsa_signature
= GNUNET_CRYPTO_rsa_sign_fdh (pk,
&hc);
GNUNET_CRYPTO_rsa_public_key_free (pub);
GNUNET_CRYPTO_rsa_private_key_free (pk);
}
{
struct TALER_Amount wire_fee;
struct TALER_MasterSignatureP master_sig;
unsigned int year;
struct GNUNET_TIME_Absolute ws;
struct GNUNET_TIME_Absolute we;
year = GNUNET_TIME_get_current_year ();
for (unsigned int y = year - 1; y<year + 2; y++)
{
ws = GNUNET_TIME_year_to_time (y - 1);
we = GNUNET_TIME_year_to_time (y);
make_amount (0, 5, &wire_fee);
memset (&master_sig,
0,
sizeof (master_sig));
if (0 >
plugin->insert_wire_fee (plugin->cls,
"aggregator-benchmark",
ws,
we,
&wire_fee,
&wire_fee,
&master_sig))
{
GNUNET_break (0);
GNUNET_SCHEDULER_shutdown ();
global_ret = EXIT_FAILURE;
return;
}
}
}
json_wire = GNUNET_JSON_PACK (
GNUNET_JSON_pack_string ("payto_uri",
"payto://aggregator-benchmark/accountfoo"),
GNUNET_JSON_pack_string ("salt",
"thesalty"));
task = GNUNET_SCHEDULER_add_now (&work,
NULL);
}
/**
* The main function of the taler-aggregator-benchmark tool.
*
* @param argc number of arguments from the command line
* @param argv command line arguments
* @return 0 ok, non-zero on failure
*/
int
main (int argc,
char *const *argv)
{
struct GNUNET_GETOPT_CommandLineOption options[] = {
GNUNET_GETOPT_option_uint ('d',
"deposits",
"DN",
"How many deposits we should instantiate per merchant",
&howmany_deposits),
GNUNET_GETOPT_option_uint ('m',
"merchants",
"DM",
"How many merchants should we create",
&howmany_merchants),
GNUNET_GETOPT_option_uint ('r',
"refunds",
"RATE",
"Probability of refund per deposit (0-100)",
&refund_rate),
GNUNET_GETOPT_OPTION_END
};
enum GNUNET_GenericReturnValue result;
unsetenv ("XDG_DATA_HOME");
unsetenv ("XDG_CONFIG_HOME");
if (0 >=
(result = GNUNET_PROGRAM_run (argc,
argv,
"taler-aggregator-benchmark",
"generate database to benchmark the aggregator",
options,
&run,
NULL)))
{
if (GNUNET_NO == result)
return EXIT_SUCCESS;
return EXIT_INVALIDARGUMENT;
}
return global_ret;
}

31
src/exchange-tools/taler-exchange-dbinit.c
View File

@ -89,20 +89,33 @@ run (void *cls,
global_ret = EXIT_NOPERMISSION;
return;
}
if (clear_shards)
if (gc_db || clear_shards)
{
if (0 < plugin->delete_revolving_shards (plugin->cls))
if (GNUNET_OK !=
plugin->preflight (plugin->cls))
{
fprintf (stderr,
"Clearing revolving shards failed!\n");
"Failed to prepare database.\n");
TALER_EXCHANGEDB_plugin_unload (plugin);
global_ret = EXIT_NOPERMISSION;
return;
}
}
if (gc_db)
{
if (GNUNET_SYSERR == plugin->gc (plugin->cls))
if (clear_shards)
{
fprintf (stderr,
"Garbage collection failed!\n");
if (0 >
plugin->delete_revolving_shards (plugin->cls))
{
fprintf (stderr,
"Clearing revolving shards failed!\n");
}
}
if (gc_db)
{
if (GNUNET_SYSERR == plugin->gc (plugin->cls))
{
fprintf (stderr,
"Garbage collection failed!\n");
}
}
}
TALER_EXCHANGEDB_plugin_unload (plugin);

15
src/exchange/taler-exchange-aggregator.c
View File

@ -124,7 +124,7 @@ struct Shard
uint32_t shard_start;
/**
* Exclusive end row of the shard.
* Inclusive end row of the shard.
*/
uint32_t shard_end;
@ -724,7 +724,7 @@ run_aggregation (void *cls)
qs = db_plugin->get_ready_deposit (
db_plugin->cls,
s->shard_start,
s->shard_end - 1, /* -1: exclusive->inclusive */
s->shard_end,
&deposit_cb,
&au_active);
switch (qs)
@ -754,9 +754,12 @@ run_aggregation (void *cls)
cleanup_au (&au_active);
db_plugin->rollback (db_plugin->cls);
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Completed shard after %s\n",
"Completed shard [%u,%u] after %s with %llu deposits\n",
(unsigned int) s->shard_start,
(unsigned int) s->shard_end,
GNUNET_STRINGS_relative_time_to_string (duration,
GNUNET_YES));
GNUNET_YES),
(unsigned long long) counter);
release_shard (s);
if (GNUNET_YES == test_mode)
{
@ -1035,6 +1038,10 @@ run_shard (void *cls)
GNUNET_SCHEDULER_shutdown ();
return;
}
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Starting shard [%u:%u]!\n",
(unsigned int) s->shard_start,
(unsigned int) s->shard_end);
task = GNUNET_SCHEDULER_add_now (&run_aggregation,
s);
}

8
src/exchangedb/plugin_exchangedb_postgres.c
View File

@ -10488,7 +10488,7 @@ postgres_complete_shard (void *cls,
* @param shard_size desired shard size
* @param shard_limit exclusive end of the shard range
* @param[out] start_row inclusive start row of the shard (returned)
* @param[out] end_row exclusive end row of the shard (returned)
* @param[out] end_row inclusive end row of the shard (returned)
* @return transaction status code
*/
static enum GNUNET_DB_QueryStatus
@ -10517,13 +10517,14 @@ postgres_begin_revolving_shard (void *cls,
/* First, find last 'end_row' */
{
enum GNUNET_DB_QueryStatus qs;
uint32_t last_end;
struct GNUNET_PQ_QueryParam params[] = {
GNUNET_PQ_query_param_string (job_name),
GNUNET_PQ_query_param_end
};
struct GNUNET_PQ_ResultSpec rs[] = {
GNUNET_PQ_result_spec_uint32 ("end_row",
start_row),
&last_end),
GNUNET_PQ_result_spec_end
};
@ -10541,6 +10542,7 @@ postgres_begin_revolving_shard (void *cls,
postgres_rollback (pg);
continue;
case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
*start_row = 1U + last_end;
break;
case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
*start_row = 0; /* base-case: no shards yet */
@ -10562,7 +10564,7 @@ postgres_begin_revolving_shard (void *cls,
};
*end_row = GNUNET_MIN (shard_limit,
*start_row + shard_size);
*start_row + shard_size - 1);
now = GNUNET_TIME_absolute_get ();
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Trying to claim shard %llu-%llu\n",

6
src/include/taler_exchangedb_plugin.h
View File

@ -974,7 +974,7 @@ struct TALER_EXCHANGEDB_Deposit
struct TALER_MerchantPublicKeyP merchant_pub;
/**
* Hash over the proposa data between merchant and customer
* Hash over the proposal data between merchant and customer
* (remains unknown to the Exchange).
*/
struct GNUNET_HashCode h_contract_terms;
@ -3760,7 +3760,7 @@ struct TALER_EXCHANGEDB_Plugin
* @param shard_size desired shard size
* @param shard_limit exclusive end of the shard range
* @param[out] start_row inclusive start row of the shard (returned)
* @param[out] end_row exclusive end row of the shard (returned)
* @param[out] end_row inclusive end row of the shard (returned)
* @return transaction status code
*/
enum GNUNET_DB_QueryStatus
@ -3779,7 +3779,7 @@ struct TALER_EXCHANGEDB_Plugin
* @param cls the @e cls of this struct with the plugin-specific state
* @param job_name name of the operation to grab a word shard for
* @param start_row inclusive start row of the shard
* @param end_row exclusive end row of the shard
* @param end_row inclusive end row of the shard
* @return transaction status code
*/
enum GNUNET_DB_QueryStatus

7
src/json/json_wire.c
View File

@ -37,9 +37,10 @@ TALER_JSON_merchant_wire_signature_hash (const json_t *wire_s,
struct GNUNET_HashCode *hc)
{
const char *payto_uri;
const char *salt; /* Current merchant backend will always make the salt
a `struct GNUNET_HashCode`, but *we* do not insist
on that. */
const char *salt;
/* Current merchant backend will always make the salt
a `struct GNUNET_HashCode`, but *we* do not insist
on that. */
struct GNUNET_JSON_Specification spec[] = {
GNUNET_JSON_spec_string ("payto_uri", &payto_uri),
GNUNET_JSON_spec_string ("salt", &salt),