implement #5967: have ranges in auditor reports, untested because right now tests somehow do not work

contrib/auditor-report.tex.j2

@@ -64,6 +64,72 @@ This report is based on a template licensed under the Affero General Public
 License, either version 3, or (at your option) any later version.
 The source code for the template is available at \url{https://git.taler.net/}.
 
+The report was generated by the auditor started at
+{\bf {{ data.auditor_start_time }} } and concluded at
+{\bf {{ data.auditor_end_time }} }.
+
+In that time, the auditor processed the following table ranges:
+\begin{center}
+\begin{table}
+\begin{tabular}{l|r|r}
+  Table                   & Start    & End \\ \hline \hline
+  Reserves Incoming       & {{ data.start_ppr_reserve_in_serial_id }}
+                          & {{ data.ppr_reserve_in_serial_id }} \\ \hline
+  Reserves Out (withdraw) & {{ data.start_ppr_reserve_out_serial_id }}
+                          & {{ data.ppr_reserve_out_serial_id }} \\ \hline
+  Reserves Payback        & {{ data.start_ppr_reserve_payback_serial_id }}
+                          & {{ data.ppr_reserve_payback_serial_id }} \\ \hline
+  Reserves Close          & {{ data.start_ppr_reserve_close_serial_id }}
+                          & {{ data.ppr_reserve_close_serial_id }} \\ \hline
+  Aggregation             & {{ data.start_ppa_wire_out_serial_id }}
+                          & {{ data.ppa_wire_out_serial_id }} \\ \hline
+  Coin withdraw           & {{ data.start_ppc_withdraw_serial_id }}
+                          & {{ data.ppc_withdraw_serial_id }} \\ \hline
+  Coin deposit            & {{ data.start_ppc_deposit_serial_id }}
+                          & {{ data.ppc_deposit_serial_id }} \\ \hline
+  Coin melt               & {{ data.start_ppc_melt_serial_id }}
+                          & {{ data.ppc_melt_serial_id }} \\ \hline
+  Coin refund             & {{ data.start_ppc_refund_serial_id }}
+                          & {{ data.ppc_refund_serial_id }} \\ \hline
+  Coin payback            & {{ data.start_ppc_payback_serial_id }}
+                          & {{ data.ppc_payback_serial_id }} \\ \hline
+  Coin payback refresh    & {{ data.start_ppc_payback_refresh_serial_id }}
+                          & {{ data.ppc_payback_refresh_serial_id }} \\
+\end{tabular}
+\caption{Serial number ranges of the tables processed by the audit.}
+\label{table:auditor_range}
+\end{table}
+\end{center}
+
+The report was generated by the wire auditor started at
+{\bf {{ wire.wire_auditor_start_time }} } and concluded at
+{\bf {{ wire.wire_auditor_end_time }} }.
+
+{% if wire.account_progress|length() == 0 %}
+In that time, the auditor processed NO accounts at all.
+{% else %}
+In that time, the auditor processed the following table ranges:
+\begin{center}
+\begin{longtable}{l|c|r|r}
+  Account & Table         & Start    & End \\ \hline \hline
+\endfirsthead
+  Account & Table         & Start    & End \\ \hline \hline
+\endhead
+\endfoot
+  \caption{Range of account data processed by the wire auditor.}
+  \label{table:account_range}
+\endlastfoot
+{% for item in wire.account_progress %}
+  {{ item.account }} &
+  Reserves Incoming       & {{ item.start_reserve_in }}
+                          & {{ item.end_reserve_in }} \\ \hline
+  Outgoing wire transfers & {{ item.start_wire_out }}
+                          & {{ item.end_wire_out }} \\ \hline
+{% endfor %}
+\end{longtable}
+{% endif %}
+\end{center}
+
 
 \section{Operations}
 

src/auditor/taler-auditor.c

@@ -135,6 +135,21 @@ static struct TALER_AUDITORDB_ProgressPointAggregation ppa;
  */
 static struct TALER_AUDITORDB_ProgressPointCoin ppc;
 
+/**
+ * Checkpointing our progress for reserves.
+ */
+static struct TALER_AUDITORDB_ProgressPointReserve ppr_start;
+
+/**
+ * Checkpointing our progress for aggregations.
+ */
+static struct TALER_AUDITORDB_ProgressPointAggregation ppa_start;
+
+/**
+ * Checkpointing our progress for coins.
+ */
+static struct TALER_AUDITORDB_ProgressPointCoin ppc_start;
+
 /**
  * Array of reports about denomination keys with an
  * emergency (more value deposited than withdrawn)
@@ -349,6 +364,11 @@ static json_t *report_refreshs_hanging;
  */
 static struct TALER_Amount total_refresh_hanging;
 
+/**
+ * At what time did the auditor process start?
+ */
+static struct GNUNET_TIME_Absolute start_time;
+
 
 /* ********************************* helpers *************************** */
 
@@ -1870,6 +1890,7 @@ analyze_reserves (void *cls)
   }
   else
   {
+    ppr_start = ppr;
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 _ ("Resuming reserve audit at %llu/%llu/%llu/%llu\n"),
                 (unsigned long long) ppr.last_reserve_in_serial_id,
@@ -3064,6 +3085,7 @@ analyze_aggregations (void *cls)
   }
   else
   {
+    ppa_start = ppa;
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 _ ("Resuming aggregation audit at %llu\n"),
                 (unsigned long long) ppa.last_wire_out_serial_id);
@@ -4638,6 +4660,7 @@ analyze_coins (void *cls)
   }
   else
   {
+    ppc_start = ppc;
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 _ ("Resuming coin audit at %llu/%llu/%llu/%llu/%llu\n"),
                 (unsigned long long) ppc.last_deposit_serial_id,
@@ -5173,6 +5196,7 @@ run (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Launching auditor\n");
   cfg = c;
+  start_time = GNUNET_TIME_absolute_get ();
   if (0 == GNUNET_memcmp (&zeromp,
                           &master_pub))
   {
@@ -5428,7 +5452,11 @@ run (void *cls,
                       " s:o, s:o, s:o, s:o, s:o,"
                       " s:o, s:o, s:o, s:o, s:I,"
                       " s:o, s:o, s:o, s:o, s:o,"
-                      " s:o }",
+                      " s:o, s:I, s:I, s:I, s:I,"
+                      " s:I, s:I, s:I, s:I, s:I,"
+                      " s:I, s:I, s:I, s:I, s:I,"
+                      " s:I, s:I, s:I, s:I, s:I,"
+                      " s:I, s:I, s:I, s:o, s:o }",
                       /* blocks of 5 for easier counting/matching to format string */
                       /* block */
                       "reserve_balance_insufficient_inconsistencies",
@@ -5542,7 +5570,61 @@ run (void *cls,
                       /* block */
                       /* Tested in test-auditor.sh #18 */
                       "emergencies_loss_by_count",
-                      TALER_JSON_from_amount (&reported_emergency_loss_by_count)
+                      TALER_JSON_from_amount (
+                        &reported_emergency_loss_by_count),
+                      "start_ppr_reserve_in_serial_id",
+                      (json_int_t) ppr_start.last_reserve_in_serial_id,
+                      "start_ppr_reserve_out_serial_id",
+                      (json_int_t) ppr_start.last_reserve_out_serial_id,
+                      "start_ppr_reserve_payback_serial_id",
+                      (json_int_t) ppr_start.last_reserve_payback_serial_id,
+                      "start_ppr_reserve_close_serial_id",
+                      (json_int_t) ppr_start.last_reserve_close_serial_id,
+                      /* block */
+                      "end_ppr_reserve_in_serial_id",
+                      (json_int_t) ppr.last_reserve_in_serial_id,
+                      "end_ppr_reserve_out_serial_id",
+                      (json_int_t) ppr.last_reserve_out_serial_id,
+                      "end_ppr_reserve_payback_serial_id",
+                      (json_int_t) ppr.last_reserve_payback_serial_id,
+                      "end_ppr_reserve_close_serial_id",
+                      (json_int_t) ppr.last_reserve_close_serial_id,
+                      "start_ppa_wire_out_serial_id",
+                      (json_int_t) ppa_start.last_wire_out_serial_id,
+                      /* block */
+                      "end_ppa_wire_out_serial_id",
+                      (json_int_t) ppa.last_wire_out_serial_id,
+                      "start_ppc_withdraw_serial_id",
+                      (json_int_t) ppc_start.last_withdraw_serial_id,
+                      "start_ppc_deposit_serial_id",
+                      (json_int_t) ppc_start.last_deposit_serial_id,
+                      "start_ppc_melt_serial_id",
+                      (json_int_t) ppc_start.last_melt_serial_id,
+                      "start_ppc_refund_serial_id",
+                      (json_int_t) ppc_start.last_refund_serial_id,
+                      /* block */
+                      "start_ppc_payback_serial_id",
+                      (json_int_t) ppc_start.last_payback_serial_id,
+                      "start_ppc_payback_refresh_serial_id",
+                      (json_int_t) ppc_start.last_payback_refresh_serial_id,
+                      "end_ppc_withdraw_serial_id",
+                      (json_int_t) ppc.last_withdraw_serial_id,
+                      "end_ppc_deposit_serial_id",
+                      (json_int_t) ppc.last_deposit_serial_id,
+                      "end_ppc_melt_serial_id",
+                      (json_int_t) ppc.last_melt_serial_id,
+                      /* block */
+                      "end_ppc_refund_serial_id",
+                      (json_int_t) ppc.last_refund_serial_id,
+                      "end_ppc_payback_serial_id",
+                      (json_int_t) ppc.last_payback_serial_id,
+                      "end_ppc_payback_refresh_serial_id",
+                      (json_int_t) ppc.last_payback_refresh_serial_id,
+                      "auditor_start_time", json_string (
+                        GNUNET_STRINGS_absolute_time_to_string (start_time)),
+                      "auditor_end_time", json_string (
+                        GNUNET_STRINGS_absolute_time_to_string (
+                          GNUNET_TIME_absolute_get ()))
                       );
   GNUNET_break (NULL != report);
   json_dumpf (report,

src/auditor/taler-wire-auditor.c

@@ -84,6 +84,11 @@ struct WireAccount
    */
   struct TALER_AUDITORDB_WireAccountProgressPoint pp;
 
+  /**
+   * Initial progress point for this account.
+   */
+  struct TALER_AUDITORDB_WireAccountProgressPoint start_pp;
+
   /**
    * Where we are in the inbound (CREDIT) transaction history.
    */
@@ -159,6 +164,11 @@ static int global_ret;
  */
 static int restart;
 
+/**
+ * Time when we started the wire audit.
+ */
+static struct GNUNET_TIME_Absolute start_time;
+
 /**
  * Handle to access the exchange's database.
  */
@@ -227,6 +237,11 @@ static enum GNUNET_DB_QueryStatus qsx;
  */
 static struct TALER_AUDITORDB_WireProgressPoint pp;
 
+/**
+ * Last reserve_in / wire_out serial IDs seen.
+ */
+static struct TALER_AUDITORDB_WireProgressPoint start_pp;
+
 /**
  * Array of reports about row inconsitencies in wire_out table.
  */
@@ -269,6 +284,11 @@ static json_t *report_lags;
  */
 static json_t *report_closure_lags;
 
+/**
+ * Array of per-account progress data.
+ */
+static json_t *report_account_progress;
+
 /**
  * Amount that is considered "tiny"
  */
@@ -483,7 +503,8 @@ do_shutdown (void *cls)
     report = json_pack ("{s:o, s:o, s:o, s:o, s:o,"
                         " s:o, s:o, s:o, s:o, s:o,"
                         " s:o, s:o, s:o, s:o, s:o,"
-                        " s:o }",
+                        " s:o, s:o, s:o, s:I, s:I,"
+                        " s:s, s:s, s:o }",
                         /* blocks of 5 */
                         /* Tested in test-auditor.sh #11, #15, #20 */
                         "wire_out_amount_inconsistencies",
@@ -533,7 +554,28 @@ do_shutdown (void *cls)
                         /* blocks of 5 */
                         /* Tested in test-auditor.sh #22 */
                         "reserve_lag_details",
-                        report_closure_lags);
+                        report_closure_lags,
+                        "wire_auditor_start_time", json_string (
+                          GNUNET_STRINGS_absolute_time_to_string (start_time)),
+                        "wire_auditor_end_time", json_string (
+                          GNUNET_STRINGS_absolute_time_to_string (
+                            GNUNET_TIME_absolute_get ())),
+                        "start_pp_reserve_close_uuid",
+                        (json_int_t) start_pp.last_reserve_close_uuid,
+                        "end_pp_reserve_close_uuid",
+                        (json_int_t) pp.last_reserve_close_uuid,
+                        /* blocks of 5 */
+                        "start_pp_last_timestamp",
+                        json_string (
+                          GNUNET_STRINGS_absolute_time_to_string (
+                            start_pp.last_timestamp)),
+                        "end_pp_last_timestamp",
+                        json_string (
+                          GNUNET_STRINGS_absolute_time_to_string (
+                            pp.last_timestamp)),
+                        "account_progress",
+                        report_account_progress
+                        );
     GNUNET_break (NULL != report);
     json_dumpf (report,
                 stdout,
@@ -546,6 +588,7 @@ do_shutdown (void *cls)
     report_missattribution_in_inconsistencies = NULL;
     report_lags = NULL;
     report_closure_lags = NULL;
+    report_account_progress = NULL;
     report_wire_format_inconsistencies = NULL;
   }
   if (NULL != reserve_closures)
@@ -717,6 +760,26 @@ commit (enum GNUNET_DB_QueryStatus qs)
        NULL != wa;
        wa = wa->next)
   {
+    GNUNET_assert (0 ==
+                   json_array_append_new (report_account_progress,
+                                          json_pack (
+                                            "{s:s, s:I, s:I, s:I, s:I}",
+                                            "account",
+                                            wa->section_name,
+                                            "start_reserve_in",
+                                            (json_int_t) wa->start_pp.
+                                            last_reserve_in_serial_id,
+                                            "end_reserve_in",
+                                            (json_int_t) wa->pp.
+                                            last_reserve_in_serial_id,
+                                            "start_wire_out",
+                                            (json_int_t) wa->start_pp.
+                                            last_wire_out_serial_id,
+                                            "end_wire_out",
+                                            (json_int_t) wa->pp.
+                                            last_wire_out_serial_id
+                                            ))
+                   );
     if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsx)
       qs = adb->update_wire_auditor_account_progress (adb->cls,
                                                       asession,
@@ -1983,6 +2046,7 @@ begin_transaction ()
       GNUNET_SCHEDULER_shutdown ();
       return;
     }
+    wa->start_pp = wa->pp;
   }
   qsx = adb->get_wire_auditor_progress (adb->cls,
                                         asession,
@@ -2003,6 +2067,7 @@ begin_transaction ()
   }
   else
   {
+    start_pp = pp;
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Resuming audit at %s / %llu\n",
                 GNUNET_STRINGS_absolute_time_to_string (pp.last_timestamp),
@@ -2228,6 +2293,8 @@ run (void *cls,
                  (report_lags = json_array ()));
   GNUNET_assert (NULL !=
                  (report_closure_lags = json_array ()));
+  GNUNET_assert (NULL !=
+                 (report_account_progress = json_array ()));
   GNUNET_assert (GNUNET_OK ==
                  TALER_amount_get_zero (currency,
                                         &total_bad_amount_out_plus));