fix aggregator fee calculation logic

This commit is contained in:
Christian Grothoff 2020-01-19 02:28:06 +01:00
parent 021a2003d5
commit 58e5780917
No known key found for this signature in database
GPG Key ID: 939E6BE1E29FC3CC
8 changed files with 828 additions and 630 deletions

View File

@ -574,17 +574,21 @@ any effects on its own balance, those entries are excluded from the total.
{% if data.coin_inconsistencies|length() == 0 %}
{\bf All coin histories were unproblematic.}
{% else %}
\begin{longtable}{p{1.8cm}|p{3cm}|r|r}
{\bf Operation} & {\bf Coin public key} & {\bf Exchange } & {\bf Auditor} \\
\begin{longtable}{p{1.8cm}|r|r}
{\bf Operation} & \multicolumn{2}{|c}{\bf Coin public key} \\
& {\bf Exchange } & {\bf Auditor} \\
\hline \hline
\endfirsthead
{\bf Operation} & {\bf Coin public key} & {\bf Exchange} & {\bf Auditor} \\ \hline \hline
{\bf Operation} & \multicolumn{2}{|r}{\bf Coin public key} \\
& {\bf Exchange } & {\bf Auditor} \\
\hline \hline
\endhead
\hline \hline
{\bf Operation} & {\bf Coin public key} & {\bf Exchange} & {\bf Auditor} \\
{\bf Operation} & \multicolumn{2}{|r}{\bf Coin public key} \\
& {\bf Exchange } & {\bf Auditor} \\
\endfoot
\hline
\multicolumn{2}{l|}{ $\sum$ {\bf Delta (Auditor-Exchange)} } &
$\sum$ {\bf Delta (Auditor-Exchange)} &
{{ data.total_coin_delta_plus }} &
- {{ data.total_coin_delta_minus }} \\
\caption{Arithmetic inconsistencies of amount calculations involving a coin.}
@ -592,8 +596,8 @@ any effects on its own balance, those entries are excluded from the total.
\endlastfoot
{% for item in data.coin_inconsistencies %}
{{ item.operation }} &
\multicolumn{5}{l}{ {\tt \small {{ item.coin_pub }} } } \\
\nopagebreak & &
\multicolumn{2}{l}{ {\tt \small {{ item.coin_pub }} } } \\
\nopagebreak &
{{ item.exchange }} &
{{ item.auditor }} \\ \hline
{% endfor %}

Binary file not shown.

View File

@ -1 +1 @@
0RT5D836NMF314QVZDSRMXX74SEPJZBSVQWF1JGBKAGGC473FJ8G
ETY0VREGBRSMW9T3H9XCDX5NQ3VZZTQ410GSXXGZWSAXEEBKKGM0

File diff suppressed because it is too large Load Diff

View File

@ -96,14 +96,14 @@ HONOR_default = YES
ACTIVE_default = YES
[fees-x-taler-bank]
wire-fee-2019 = TESTKUDOS:0.01
closing-fee-2019 = TESTKUDOS:0.01
wire-fee-2020 = TESTKUDOS:0.01
closing-fee-2020 = TESTKUDOS:0.01
wire-fee-2021 = TESTKUDOS:0.01
closing-fee-2021 = TESTKUDOS:0.01
wire-fee-2022 = TESTKUDOS:0.01
closing-fee-2022 = TESTKUDOS:0.01
wire-fee-2019 = TESTKUDOS:0.10
closing-fee-2019 = TESTKUDOS:0.20
wire-fee-2020 = TESTKUDOS:0.10
closing-fee-2020 = TESTKUDOS:0.20
wire-fee-2021 = TESTKUDOS:0.10
closing-fee-2021 = TESTKUDOS:0.20
wire-fee-2022 = TESTKUDOS:0.10
closing-fee-2022 = TESTKUDOS:0.20
[merchant-instance-wireformat-default]
TEST_RESPONSE_FILE = ${TALER_CONFIG_HOME}/merchant/wire/tutorial.json
@ -128,9 +128,9 @@ duration_withdraw = 7 days
duration_spend = 2 years
duration_legal = 3 years
fee_withdraw = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.01
fee_refresh = TESTKUDOS:0.01
fee_refund = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.03
fee_refresh = TESTKUDOS:0.05
fee_refund = TESTKUDOS:0.07
rsa_keysize = 1024
[coin_kudos_ct_10]
@ -140,9 +140,9 @@ duration_withdraw = 7 days
duration_spend = 2 years
duration_legal = 3 years
fee_withdraw = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.01
fee_refresh = TESTKUDOS:0.03
fee_refund = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.03
fee_refresh = TESTKUDOS:0.05
fee_refund = TESTKUDOS:0.07
rsa_keysize = 1024
[coin_kudos_1]
@ -151,10 +151,10 @@ duration_overlap = 5 minutes
duration_withdraw = 7 days
duration_spend = 2 years
duration_legal = 3 years
fee_withdraw = TESTKUDOS:0.02
fee_deposit = TESTKUDOS:0.02
fee_refresh = TESTKUDOS:0.03
fee_refund = TESTKUDOS:0.01
fee_withdraw = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.03
fee_refresh = TESTKUDOS:0.05
fee_refund = TESTKUDOS:0.07
rsa_keysize = 1024
[coin_kudos_2]
@ -163,10 +163,10 @@ duration_overlap = 5 minutes
duration_withdraw = 7 days
duration_spend = 2 years
duration_legal = 3 years
fee_withdraw = TESTKUDOS:0.03
fee_withdraw = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.03
fee_refresh = TESTKUDOS:0.04
fee_refund = TESTKUDOS:0.02
fee_refresh = TESTKUDOS:0.05
fee_refund = TESTKUDOS:0.07
rsa_keysize = 1024
[coin_kudos_4]
@ -175,10 +175,10 @@ duration_overlap = 5 minutes
duration_withdraw = 7 days
duration_spend = 2 years
duration_legal = 3 years
fee_withdraw = TESTKUDOS:0.03
fee_withdraw = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.03
fee_refresh = TESTKUDOS:0.04
fee_refund = TESTKUDOS:0.02
fee_refresh = TESTKUDOS:0.05
fee_refund = TESTKUDOS:0.07
rsa_keysize = 1024
[coin_kudos_5]
@ -188,9 +188,9 @@ duration_withdraw = 7 days
duration_spend = 2 years
duration_legal = 3 years
fee_withdraw = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.01
fee_refresh = TESTKUDOS:0.03
fee_refund = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.03
fee_refresh = TESTKUDOS:0.05
fee_refund = TESTKUDOS:0.07
rsa_keysize = 1024
[coin_kudos_8]
@ -199,10 +199,10 @@ duration_overlap = 5 minutes
duration_withdraw = 7 days
duration_spend = 2 years
duration_legal = 3 years
fee_withdraw = TESTKUDOS:0.05
fee_deposit = TESTKUDOS:0.02
fee_refresh = TESTKUDOS:0.03
fee_refund = TESTKUDOS:0.04
fee_withdraw = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.03
fee_refresh = TESTKUDOS:0.05
fee_refund = TESTKUDOS:0.07
rsa_keysize = 1024
[coin_kudos_10]
@ -212,9 +212,9 @@ duration_withdraw = 7 days
duration_spend = 2 years
duration_legal = 3 years
fee_withdraw = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.01
fee_refresh = TESTKUDOS:0.03
fee_refund = TESTKUDOS:0.01
fee_deposit = TESTKUDOS:0.03
fee_refresh = TESTKUDOS:0.05
fee_refund = TESTKUDOS:0.07
rsa_keysize = 1024
[benchmark]

View File

@ -2151,6 +2151,7 @@ struct WireCheckContext
* @param issue denomination information about the coin
* @param tl_head head of transaction history to verify
* @param[out] merchant_gain amount the coin contributes to the wire transfer to the merchant
* @param[out] deposit_gain amount the coin contributes excluding refunds
* @return #GNUNET_OK on success, #GNUNET_SYSERR on error
*/
static int
@ -2164,14 +2165,15 @@ check_transaction_history_for_deposit (const struct
TALER_DenominationKeyValidityPS *issue,
const struct
TALER_EXCHANGEDB_TransactionList *tl_head,
struct TALER_Amount *merchant_gain)
struct TALER_Amount *merchant_gain,
struct TALER_Amount *deposit_gain)
{
struct TALER_Amount expenditures;
struct TALER_Amount refunds;
struct TALER_Amount spent;
struct TALER_Amount value;
struct TALER_Amount merchant_loss;
struct TALER_Amount merchant_delta;
struct TALER_Amount final_gain;
const struct TALER_Amount *deposit_fee;
int refund_deposit_fee;
@ -2336,6 +2338,9 @@ check_transaction_history_for_deposit (const struct
(0 == GNUNET_memcmp (h_contract_terms,
&tl->details.refund->h_contract_terms)) )
{
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Detected applicable refund of %s\n",
TALER_amount2s (amount_with_fee));
if (GNUNET_OK !=
TALER_amount_add (&merchant_loss,
&merchant_loss,
@ -2344,9 +2349,6 @@ check_transaction_history_for_deposit (const struct
GNUNET_break (0);
return GNUNET_SYSERR;
}
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Detected applicable refund of %s\n",
TALER_amount2s (amount_with_fee));
refund_deposit_fee = GNUNET_YES;
}
/* Check that the fees given in the transaction list and in dki match */
@ -2397,19 +2399,15 @@ check_transaction_history_for_deposit (const struct
}
} /* for 'tl' */
if ( (GNUNET_YES == refund_deposit_fee) &&
(NULL != deposit_fee) )
{
/* We had a /deposit operation AND a /refund operation,
and should thus not charge the merchant the /deposit fee */
GNUNET_assert (GNUNET_OK ==
TALER_amount_add (merchant_gain,
merchant_gain,
deposit_fee));
}
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Deposits without fees are %s\n",
TALER_amount2s (merchant_gain));
/* Calculate total balance change, i.e. expenditures (recoup, deposit, refresh)
minus refunds (refunds, recoup-to-old) */
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Subtracting refunds of %s from coin value loss\n",
TALER_amount2s (&refunds));
if (GNUNET_SYSERR ==
TALER_amount_subtract (&spent,
&expenditures,
@ -2441,8 +2439,22 @@ check_transaction_history_for_deposit (const struct
/* Finally, update @a merchant_gain by subtracting what he "lost"
from refunds */
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Merchant 'loss' due to refunds is %s\n",
TALER_amount2s (&merchant_loss));
*deposit_gain = *merchant_gain;
if ( (GNUNET_YES == refund_deposit_fee) &&
(NULL != deposit_fee) )
{
/* We had a /deposit operation AND a /refund operation,
and should thus not charge the merchant the /deposit fee */
GNUNET_assert (GNUNET_OK ==
TALER_amount_add (merchant_gain,
merchant_gain,
deposit_fee));
}
if (GNUNET_SYSERR ==
TALER_amount_subtract (&merchant_delta,
TALER_amount_subtract (&final_gain,
merchant_gain,
&merchant_loss))
{
@ -2454,7 +2466,10 @@ check_transaction_history_for_deposit (const struct
1);
return GNUNET_SYSERR;
}
*merchant_gain = merchant_delta;
*merchant_gain = final_gain;
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Final merchant gain after refunds is %s\n",
TALER_amount2s (deposit_gain));
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Coin %s contributes %s to contract %s\n",
TALER_B2S (coin_pub),
@ -2477,7 +2492,8 @@ check_transaction_history_for_deposit (const struct
* @param h_contract_terms which proposal was this payment about
* @param denom_pub denomination of @a coin_pub
* @param coin_pub which public key was this payment about
* @param coin_value amount contributed by this coin in total (with fee)
* @param coin_value amount contributed by this coin in total (with fee),
* but excluding refunds by this coin
* @param deposit_fee applicable deposit fee for this coin, actual
* fees charged may differ if coin was refunded
*/
@ -2500,6 +2516,7 @@ wire_transfer_information_cb (void *cls,
const struct TALER_DenominationKeyValidityPS *issue;
struct TALER_Amount computed_value;
struct TALER_Amount coin_value_without_fee;
struct TALER_Amount total_deposit_without_refunds;
struct TALER_EXCHANGEDB_TransactionList *tl;
struct TALER_CoinPublicInfo coin;
enum GNUNET_DB_QueryStatus qs;
@ -2600,7 +2617,8 @@ wire_transfer_information_cb (void *cls,
merchant_pub,
issue,
tl,
&computed_value))
&computed_value,
&total_deposit_without_refunds))
{
wcc->qs = GNUNET_DB_STATUS_HARD_ERROR;
report_row_inconsistency ("coin history",
@ -2608,6 +2626,9 @@ wire_transfer_information_cb (void *cls,
"failed to verify coin history (for deposit)");
return;
}
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Coin contributes %s to aggregate (deposits after fees and refunds)\n",
TALER_amount2s (&computed_value));
if (GNUNET_SYSERR ==
TALER_amount_subtract (&coin_value_without_fee,
coin_value,
@ -2622,14 +2643,17 @@ wire_transfer_information_cb (void *cls,
return;
}
if (0 !=
TALER_amount_cmp (&computed_value,
TALER_amount_cmp (&total_deposit_without_refunds,
&coin_value_without_fee))
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Expected coin contribution of %s to aggregate\n",
TALER_amount2s (&coin_value_without_fee));
wcc->qs = GNUNET_DB_STATUS_HARD_ERROR;
report_amount_arithmetic_inconsistency ("aggregation (contribution)",
rowid,
&coin_value_without_fee,
&computed_value,
&total_deposit_without_refunds,
-1);
}
edb->free_coin_transaction_list (edb->cls,
@ -2661,7 +2685,7 @@ wire_transfer_information_cb (void *cls,
if (GNUNET_OK !=
TALER_amount_add (&res,
&wcc->total_deposits,
&coin_value_without_fee))
&computed_value))
{
GNUNET_break (0);
wcc->qs = GNUNET_DB_STATUS_HARD_ERROR;
@ -2797,13 +2821,12 @@ get_wire_fee (struct AggregationContext *ac,
* @return #GNUNET_OK to continue, #GNUNET_SYSERR to stop iteration
*/
static int
check_wire_out_cb
(void *cls,
uint64_t rowid,
struct GNUNET_TIME_Absolute date,
const struct TALER_WireTransferIdentifierRawP *wtid,
const json_t *wire,
const struct TALER_Amount *amount)
check_wire_out_cb (void *cls,
uint64_t rowid,
struct GNUNET_TIME_Absolute date,
const struct TALER_WireTransferIdentifierRawP *wtid,
const json_t *wire,
const struct TALER_Amount *amount)
{
struct AggregationContext *ac = cls;
struct WireCheckContext wcc;

View File

@ -151,6 +151,7 @@ jq -e .row_minor_inconsistencies[0] < test-wire-audit.json > /dev/null && exit_f
jq -e .lag_details[0] < test-wire-audit.json > /dev/null && exit_fail "Unexpected lag detected in ordinary run"
jq -e .wire_format_inconsistencies[0] < test-wire-audit.json > /dev/null && exit_fail "Unexpected wire format inconsistencies detected in ordinary run"
# FIXME: check operation balances are correct (once we have more transaction types)
# FIXME: check revenue summaries are correct (once we have more transaction types)
@ -188,7 +189,26 @@ if test $WIRED != "TESTKUDOS:0"
then
exit_fail "Expected total missattribution in wrong, got $WIRED"
fi
echo " OK"
echo PASS
echo -n "Checking for unexpected arithmetic differences "
LOSS=`jq -r .total_arithmetic_delta_plus < test-audit.json`
if test $LOSS != "TESTKUDOS:0"
then
exit_fail "Wrong arithmetic delta, got unexpected plus of $LOSS"
fi
LOSS=`jq -r .total_arithmetic_delta_minus < test-audit.json`
if test $LOSS != "TESTKUDOS:0"
then
exit_fail "Wrong arithmetic delta, got unexpected minus of $LOSS"
fi
jq -e .amount_arithmetic_inconsistencies[0] < test-audit.json > /dev/null && exit_fail "Unexpected arithmetic inconsistencies detected in ordinary run"
echo PASS
echo -n "Checking for unexpected wire out differences "
jq -e .wire_out_inconsistencies[0] < test-audit.json > /dev/null && exit_fail "Unexpected wire out inconsistencies detected in ordinary run"
echo PASS
# FIXME: check NO lag reported
@ -331,13 +351,13 @@ echo "UPDATE reserves_in SET credit_val=15 WHERE reserve_in_serial_id=1" | psql
run_audit
EXPECTED=`jq -r .reserve_balance_summary_wrong_inconsistencies[0].auditor < test-audit.json`
if test $EXPECTED != "TESTKUDOS:5.01"
if test $EXPECTED != "TESTKUDOS:5"
then
exit_fail "Expected reserve balance summary amount wrong, got $EXPECTED (auditor)"
fi
EXPECTED=`jq -r .reserve_balance_summary_wrong_inconsistencies[0].exchange < test-audit.json`
if test $EXPECTED != "TESTKUDOS:0.01"
if test $EXPECTED != "TESTKUDOS:0"
then
exit_fail "Expected reserve balance summary amount wrong, got $EXPECTED (exchange)"
fi
@ -493,7 +513,7 @@ then
fi
LOSS=`jq -r .bad_sig_losses[0].loss < test-audit.json`
if test $LOSS != "TESTKUDOS:0.1"
if test $LOSS != "TESTKUDOS:1"
then
exit_fail "Wrong deposit bad signature loss, got $LOSS"
fi
@ -505,7 +525,7 @@ then
fi
LOSS=`jq -r .total_bad_sig_loss < test-audit.json`
if test $LOSS != "TESTKUDOS:0.1"
if test $LOSS != "TESTKUDOS:1"
then
exit_fail "Wrong total bad sig loss, got $LOSS"
fi

View File

@ -179,6 +179,11 @@ struct AggregationUnit
*/
int failed;
/**
* Set to #GNUNET_YES if we encountered a refund during #refund_by_coin_cb.
* Used to wave the deposit fee.
*/
int have_refund;
};
@ -670,6 +675,7 @@ refund_by_coin_cb (void *cls,
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Aggregator subtracts applicable refund of amount %s\n",
TALER_amount2s (amount_with_fee));
aux->have_refund = GNUNET_YES;
if (GNUNET_OK !=
TALER_amount_subtract (&aux->total_amount,
&aux->total_amount,
@ -716,25 +722,13 @@ deposit_cb (void *cls,
fetch this one: */
(void) wire_deadline; /* already checked by SQL query */
au->merchant_pub = *merchant_pub;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Discovered ready transaction, starting by subtracting deposit fee %s\n",
TALER_amount2s (deposit_fee));
if (GNUNET_SYSERR ==
TALER_amount_subtract (&au->total_amount,
amount_with_fee,
deposit_fee))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Fatally malformed record at row %llu over %s\n",
(unsigned long long) row_id,
TALER_amount2s (amount_with_fee));
return GNUNET_DB_STATUS_HARD_ERROR;
}
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Aggregator processing payment %s with amount %s after fee subtraction\n",
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Aggregator processing payment %s with amount %s\n",
TALER_B2S (coin_pub),
TALER_amount2s (&au->total_amount));
TALER_amount2s (amount_with_fee));
au->row_id = row_id;
au->total_amount = *amount_with_fee;
au->have_refund = GNUNET_NO;
qs = db_plugin->select_refunds_by_coin (db_plugin->cls,
au->session,
coin_pub,
@ -747,6 +741,23 @@ deposit_cb (void *cls,
GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
return qs;
}
if (GNUNET_NO == au->have_refund)
{
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Non-refunded transaction, subtracting deposit fee %s\n",
TALER_amount2s (deposit_fee));
if (GNUNET_SYSERR ==
TALER_amount_subtract (&au->total_amount,
amount_with_fee,
deposit_fee))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Fatally malformed record at row %llu over %s\n",
(unsigned long long) row_id,
TALER_amount2s (amount_with_fee));
return GNUNET_DB_STATUS_HARD_ERROR;
}
}
GNUNET_assert (NULL == au->wire);
if (NULL == (au->wire = json_incref ((json_t *) wire)))
@ -870,29 +881,15 @@ aggregate_cb (void *cls,
GNUNET_break (0 == GNUNET_memcmp (&au->merchant_pub,
merchant_pub));
/* compute contribution of this coin after fees */
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Subtracting deposit fee %s\n",
TALER_amount2s (deposit_fee));
if (GNUNET_SYSERR ==
TALER_amount_subtract (&delta,
amount_with_fee,
deposit_fee))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Fatally malformed record at %llu over amount %s\n",
(unsigned long long) row_id,
TALER_amount2s (amount_with_fee));
return GNUNET_DB_STATUS_HARD_ERROR;
}
/* add to total */
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Adding transaction amount %s from row %llu to aggregation\n",
TALER_amount2s (&delta),
TALER_amount2s (amount_with_fee),
(unsigned long long) row_id);
if (GNUNET_OK !=
TALER_amount_add (&au->total_amount,
&au->total_amount,
&delta))
amount_with_fee))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Overflow or currency incompatibility during aggregation at %llu\n",
@ -900,7 +897,7 @@ aggregate_cb (void *cls,
/* Skip this one, but keep going! */
return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
}
au->have_refund = GNUNET_NO;
qs = db_plugin->select_refunds_by_coin (db_plugin->cls,
au->session,
coin_pub,
@ -913,6 +910,24 @@ aggregate_cb (void *cls,
GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
return qs;
}
if (GNUNET_NO == au->have_refund)
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Subtracting deposit fee %s for non-refunded coin\n",
TALER_amount2s (deposit_fee));
if (GNUNET_SYSERR ==
TALER_amount_subtract (&delta,
&au->total_amount,
deposit_fee))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Fatally malformed record at %llu over amount %s\n",
(unsigned long long) row_id,
TALER_amount2s (&au->total_amount));
return GNUNET_DB_STATUS_HARD_ERROR;
}
au->total_amount = delta;
}
if (au->rows_offset >= aggregation_limit)
{