implementing #4961: detection of missing aggregate transfers

contrib/auditor-report.tex.j2

@@ -59,6 +59,60 @@ This section analyzes the income of the exchange operator from fees.
 \end{table}
 
 
+\section{Lag}
+
+This section analyzes the lag, which is by how much the exchange's aggregator is behind in
+making wire transfers that have been due.
+
+The total amount the exchange currently lags behind is
+{\bf  {{ wire.total_amount_lag.value }}.{{ wire.total_amount_lag.fraction }}
+      {{ wire.total_amount_lag.currency }}
+}.
+
+Note that some lag is perfectly normal, as tiny amounts that are too small to be wired
+are deferred beyond the due date, hoping that additional transfers will push them above
+the tiny threshold. Below, we report {\em non-tiny} wire transfers that are lagging behind.
+
+{% if wire.lag_details|length() == 0 %}
+  {\bf No non-tiny wire transfers that are lagging behind detected.}
+{% else %}
+  \begin{longtable}{p{1.5cm}|rl|c|rl}
+  \multicolumn{4}{l}{\bf Coin} \\
+  {\bf Deadline} & {\bf Amount} & {\bf Row} & {\bf Claimed done} \\
+  \multicolumn{4}{l}{\bf Target account} \\ \hline \hline
+\endfirsthead
+  \multicolumn{4}{l}{\bf Coin} \\
+  {\bf Deadline} & {\bf Amount} & {\bf Row} & {\bf Claimed done} \\
+  \multicolumn{4}{l}{\bf Target account} \\ \hline \hline
+\endhead
+  \hline \hline
+  \multicolumn{4}{l}{\bf Coin} \\
+  {\bf Deadline} & {\bf Amount} & {\bf Row} & {\bf Claimed done} \\
+  \multicolumn{4}{l}{\bf Target account} \\
+\endfoot
+  \hline \hline
+  \multicolumn{4}{l}{\bf Coin} \\
+  {\bf Deadline} & {\bf Amount} & {\bf Row} & {\bf Claimed done} \\
+  \multicolumn{4}{l}{\bf Target account} \\
+    \caption{Lagging non-tiny transactions.}
+    \label{table:lag}
+\endlastfoot
+{% for item in wire.lag_details %}
+  \multicolumn{4}{l}{ {\tt {{ item.coin_pub }} } } \\
+\nopagebreak
+  &
+  {{ item.deadline }} &
+  {{ item.amount.value }}.{{ item.amount.fraction }} &
+  {{ item.amount.currency }} &
+  {{ item.row }} &
+  {{ item.claimed_done }} \\
+\nopagebreak
+  \multicolumn{4}{l}{ {\tt {{ item.account }} } } \\ \hline
+{% endfor %}
+  \end{longtable}
+{% endif %}
+
+
 \section{Major irregularities}
 
 This section describes the possible major irregularities that the

src/auditor/taler-wire-auditor.c

@@ -22,6 +22,8 @@
  *   the incoming wire transfers from the bank.
  * - Second, we check that the outgoing wire transfers match those
  *   given in the 'wire_out' table
+ * - Finally, we check that all wire transfers that should have been made,
+ *   were actually made
  */
 #include "platform.h"
 #include <gnunet/gnunet_util_lib.h>
@@ -31,6 +33,12 @@
 #include "taler_wire_lib.h"
 #include "taler_signatures.h"
 
+/**
+ * How much time do we allow the aggregator to lag behind?  If
+ * wire transfers should have been made more than #GRACE_PERIOD
+ * before, we issue warnings.
+ */
+#define GRACE_PERIOD GNUNET_TIME_UNIT_HOURS
 
 /**
  * Return value from main().
@@ -156,6 +164,11 @@ static json_t *report_row_inconsistencies;
  */
 static json_t *report_row_minor_inconsistencies;
 
+/**
+ * Array of reports about lagging transactions.
+ */
+static json_t *report_lags;
+
 /**
  * Total amount that was transferred too much from the exchange.
  */
@@ -183,6 +196,11 @@ static struct TALER_Amount total_bad_amount_in_minus;
  */
 static struct TALER_Amount total_missattribution_in;
 
+/**
+ * Total amount which the exchange did not transfer in time.
+ */
+static struct TALER_Amount total_amount_lag;
+
 /**
  * Amount of zero in our currency.
  */
@@ -305,7 +323,8 @@ do_shutdown (void *cls)
 
     GNUNET_assert (NULL != report_row_minor_inconsistencies);
     report = json_pack ("{s:o, s:o, s:o, s:o, s:o,"
-                        " s:o, s:o, s:o, s:o, s:o }",
+                        " s:o, s:o, s:o, s:o, s:o,"
+                        " s:o, s:o }",
                         /* blocks of 5 */
 			"wire_out_amount_inconsistencies",
                         report_wire_out_inconsistencies,
@@ -327,7 +346,12 @@ do_shutdown (void *cls)
 			"row_inconsistencies",
                         report_row_inconsistencies,
 			"row_minor_inconsistencies",
-                        report_row_minor_inconsistencies);
+                        report_row_minor_inconsistencies,
+                        /* block */
+                        "total_amount_lag",
+                        TALER_JSON_from_amount (&total_bad_amount_in_minus),
+                        "lag_details",
+                        report_lags);
     GNUNET_break (NULL != report);
     json_dumpf (report,
 		stdout,
@@ -338,6 +362,7 @@ do_shutdown (void *cls)
     report_row_inconsistencies = NULL;
     report_row_minor_inconsistencies = NULL;
     report_missattribution_in_inconsistencies = NULL;
+    report_lags = NULL;
   }
   if (NULL != hh)
   {
@@ -672,6 +697,57 @@ complain_out_not_found (void *cls,
 }
 
 
+/**
+ * Function called on deposits that are past their due date
+ * and have not yet seen a wire transfer.
+ *
+ * @param cls closure
+ * @param rowid deposit table row of the coin's deposit
+ * @param coin_pub public key of the coin
+ * @param amount value of the deposit, including fee
+ * @param wire where should the funds be wired
+ * @param deadline what was the requested wire transfer deadline
+ * @param tiny did the exchange defer this transfer because it is too small?
+ * @param done did the exchange claim that it made a transfer?
+ */
+static void
+wire_missing_cb (void *cls,
+                 uint64_t rowid,
+                 const struct TALER_CoinSpendPublicKeyP *coin_pub,
+                 const struct TALER_Amount *amount,
+                 const json_t *wire,
+                 struct GNUNET_TIME_Absolute deadline,
+                 /* bool? */ int tiny,
+                 /* bool? */ int done)
+{
+  GNUNET_break (GNUNET_OK ==
+                TALER_amount_add (&total_amount_lag,
+                                  &total_amount_lag,
+                                  amount));
+  if (GNUNET_YES == tiny)
+  {
+    struct TALER_Amount rounded;
+
+    rounded = *amount;
+    GNUNET_break (GNUNET_SYSERR !=
+                  wp->amount_round (wp->cls,
+                                    &rounded));
+    if (0 == TALER_amount_cmp (&rounded,
+                               &zero))
+      return; /* acceptable, amount was tiny */
+  }
+  report (report_lags,
+          json_pack ("{s:I, s:o, s:s, s:s, s:o, s:O}",
+                     "row", (json_int_t) rowid,
+                     "amount", TALER_JSON_from_amount (amount),
+                     "deadline", GNUNET_STRINGS_absolute_time_to_string (deadline),
+                     "claimed_done", (done) ? "yes" : "no",
+                     "coin_pub", GNUNET_JSON_from_data_auto (coin_pub),
+                     "account", wire));
+
+}
+
+
 /**
  * Go over the "wire_out" table of the exchange and
  * verify that all wire outs are in that table.
@@ -680,6 +756,7 @@ static void
 check_exchange_wire_out ()
 {
   enum GNUNET_DB_QueryStatus qs;
+  struct GNUNET_TIME_Absolute next_timestamp;
 
   qs = edb->select_wire_out_above_serial_id (edb->cls,
 					     esession,
@@ -703,6 +780,28 @@ check_exchange_wire_out ()
   GNUNET_CONTAINER_multihashmap_destroy (out_map);
   out_map = NULL;
 
+  /* now check that all wire transfers that should have happened,
+     have indeed happened */
+  next_timestamp = GNUNET_TIME_absolute_get ();
+  /* Subtract #GRACE_PERIOD, so we can be a bit behind in processing
+     without immediately raising undue concern */
+  next_timestamp = GNUNET_TIME_absolute_subtract (next_timestamp,
+                                                  GRACE_PERIOD);
+  qs = edb->select_deposits_missing_wire (edb->cls,
+                                          esession,
+                                          pp.last_timestamp,
+                                          next_timestamp,
+                                          &wire_missing_cb,
+                                          &next_timestamp);
+  if (0 > qs)
+  {
+    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
+    global_ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  pp.last_timestamp = next_timestamp;
+
   /* conclude with: */
   commit (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT);
   GNUNET_SCHEDULER_shutdown ();
@@ -1286,6 +1385,8 @@ run (void *cls,
 		 (report_row_inconsistencies = json_array ()));
   GNUNET_assert (NULL !=
 		 (report_missattribution_in_inconsistencies = json_array ()));
+  GNUNET_assert (NULL !=
+		 (report_lags = json_array ()));
   GNUNET_assert (GNUNET_OK ==
                  TALER_amount_get_zero (currency,
                                         &total_bad_amount_out_plus));
@@ -1301,6 +1402,9 @@ run (void *cls,
   GNUNET_assert (GNUNET_OK ==
                  TALER_amount_get_zero (currency,
                                         &total_missattribution_in));
+  GNUNET_assert (GNUNET_OK ==
+                 TALER_amount_get_zero (currency,
+                                        &total_amount_lag));
   GNUNET_assert (GNUNET_OK ==
                  TALER_amount_get_zero (currency,
                                         &zero));