working on splitting auditor

src/auditor/.gitignore

@@ -12,3 +12,4 @@ test-audit-inc.json
 test-wire-audit-inc.json
 wirefees/
 bank.err
+libauditor.a

src/auditor/Makefile.am

@@ -13,12 +13,22 @@ pkgcfg_DATA = \
 
 bin_PROGRAMS = \
   taler-auditor \
+  taler-auditor-reserves \
+  taler-auditor-coins \
+  taler-auditor-aggregation \
+  taler-auditor-deposits \
+  taler-wire-auditor \
   taler-auditor-exchange \
   taler-auditor-httpd \
-  taler-wire-auditor \
   taler-auditor-sign \
   taler-auditor-dbinit
 
+noinst_LIBRARIES = \
+  libauditor.a
+
+libauditor_a_SOURCES = \
+  report-lib.c report-lib.h
+
 taler_auditor_dbinit_SOURCES = \
   taler-auditor-dbinit.c
 taler_auditor_dbinit_LDADD = \
@@ -34,6 +44,62 @@ taler_auditor_dbinit_CPPFLAGS = \
   -I$(top_srcdir)/src/pq/ \
   $(POSTGRESQL_CPPFLAGS)
 
+taler_auditor_reserves_SOURCES = \
+  taler-auditor-reserves.c
+taler_auditor_reserves_LDADD = \
+  $(LIBGCRYPT_LIBS) \
+  $(top_builddir)/src/util/libtalerutil.la \
+  $(top_builddir)/src/json/libtalerjson.la \
+  $(top_builddir)/src/bank-lib/libtalerbank.la \
+  $(top_builddir)/src/exchangedb/libtalerexchangedb.la \
+  $(top_builddir)/src/auditordb/libtalerauditordb.la \
+  libauditor.a \
+  -ljansson \
+  -lgnunetjson \
+  -lgnunetutil
+
+taler_auditor_coins_SOURCES = \
+  taler-auditor-coins.c
+taler_auditor_coins_LDADD = \
+  $(LIBGCRYPT_LIBS) \
+  $(top_builddir)/src/util/libtalerutil.la \
+  $(top_builddir)/src/json/libtalerjson.la \
+  $(top_builddir)/src/bank-lib/libtalerbank.la \
+  $(top_builddir)/src/exchangedb/libtalerexchangedb.la \
+  $(top_builddir)/src/auditordb/libtalerauditordb.la \
+  libauditor.a \
+  -ljansson \
+  -lgnunetjson \
+  -lgnunetutil
+
+taler_auditor_aggregation_SOURCES = \
+  taler-auditor-aggregation.c
+taler_auditor_aggregation_LDADD = \
+  $(LIBGCRYPT_LIBS) \
+  $(top_builddir)/src/util/libtalerutil.la \
+  $(top_builddir)/src/json/libtalerjson.la \
+  $(top_builddir)/src/bank-lib/libtalerbank.la \
+  $(top_builddir)/src/exchangedb/libtalerexchangedb.la \
+  $(top_builddir)/src/auditordb/libtalerauditordb.la \
+  libauditor.a \
+  -ljansson \
+  -lgnunetjson \
+  -lgnunetutil
+
+taler_auditor_deposits_SOURCES = \
+  taler-auditor-deposits.c
+taler_auditor_deposits_LDADD = \
+  $(LIBGCRYPT_LIBS) \
+  $(top_builddir)/src/util/libtalerutil.la \
+  $(top_builddir)/src/json/libtalerjson.la \
+  $(top_builddir)/src/bank-lib/libtalerbank.la \
+  $(top_builddir)/src/exchangedb/libtalerexchangedb.la \
+  $(top_builddir)/src/auditordb/libtalerauditordb.la \
+  libauditor.a \
+  -ljansson \
+  -lgnunetjson \
+  -lgnunetutil
+
 taler_auditor_SOURCES = \
   taler-auditor.c
 taler_auditor_LDADD = \
@@ -47,6 +113,7 @@ taler_auditor_LDADD = \
   -lgnunetjson \
   -lgnunetutil
 
+
 taler_auditor_httpd_SOURCES = \
   taler-auditor-httpd.c taler-auditor-httpd.h \
   taler-auditor-httpd_deposit-confirmation.c taler-auditor-httpd_deposit-confirmation.h \

src/auditor/report-lib.c

@@ -0,0 +1,549 @@
+/*
+  This file is part of TALER
+  Copyright (C) 2016-2020 Taler Systems SA
+
+  TALER is free software; you can redistribute it and/or modify it under the
+  terms of the GNU Affero Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU Affero Public License for more details.
+
+  You should have received a copy of the GNU Affero Public License along with
+  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file auditor/report-lib.c
+ * @brief helper library to facilitate generation of audit reports
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "report-lib.h"
+
+/**
+ * Command-line option "-r": restart audit from scratch
+ */
+int restart;
+
+/**
+ * Handle to access the exchange's database.
+ */
+struct TALER_EXCHANGEDB_Plugin *edb;
+
+/**
+ * Which currency are we doing the audit for?
+ */
+char *currency;
+
+/**
+ * How many fractional digits does the currency use?
+ */
+struct TALER_Amount currency_round_unit;
+
+/**
+ * Our configuration.
+ */
+const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
+ * Our session with the #edb.
+ */
+struct TALER_EXCHANGEDB_Session *esession;
+
+/**
+ * Handle to access the auditor's database.
+ */
+struct TALER_AUDITORDB_Plugin *adb;
+
+/**
+ * Our session with the #adb.
+ */
+struct TALER_AUDITORDB_Session *asession;
+
+/**
+ * Master public key of the exchange to audit.
+ */
+struct TALER_MasterPublicKeyP master_pub;
+
+/**
+ * At what time did the auditor process start?
+ */
+struct GNUNET_TIME_Absolute start_time;
+
+/**
+ * Results about denominations, cached per-transaction, maps denomination pub hashes
+ * to `struct TALER_DenominationKeyValidityPS`.
+ */
+static struct GNUNET_CONTAINER_MultiHashMap *denominations;
+
+
+/**
+ * Convert absolute time to human-readable JSON string.
+ *
+ * @param at time to convert
+ * @return human-readable string representing the time
+ */
+json_t *
+json_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO at)
+{
+  return json_string
+           (GNUNET_STRINGS_absolute_time_to_string
+             (GNUNET_TIME_absolute_ntoh (at)));
+}
+
+
+/**
+ * Convert absolute time to human-readable JSON string.
+ *
+ * @param at time to convert
+ * @return human-readable string representing the time
+ */
+json_t *
+json_from_time_abs (struct GNUNET_TIME_Absolute at)
+{
+  return json_string
+           (GNUNET_STRINGS_absolute_time_to_string (at));
+}
+
+
+/**
+ * Add @a object to the report @a array.  Fail hard if this fails.
+ *
+ * @param array report array to append @a object to
+ * @param object object to append, should be check that it is not NULL
+ */
+void
+report (json_t *array,
+        json_t *object)
+{
+  GNUNET_assert (NULL != object);
+  GNUNET_assert (0 ==
+                 json_array_append_new (array,
+                                        object));
+}
+
+
+/**
+ * Function called with the results of select_denomination_info()
+ *
+ * @param cls closure, NULL
+ * @param issue issuing information with value, fees and other info about the denomination.
+ * @return #GNUNET_OK (to continue)
+ */
+static int
+add_denomination (void *cls,
+                  const struct TALER_DenominationKeyValidityPS *issue)
+{
+  struct TALER_DenominationKeyValidityPS *i;
+
+  (void) cls;
+  if (NULL !=
+      GNUNET_CONTAINER_multihashmap_get (denominations,
+                                         &issue->denom_hash))
+    return GNUNET_OK; /* value already known */
+  {
+    struct TALER_Amount value;
+
+    TALER_amount_ntoh (&value,
+                       &issue->value);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Tracking denomination `%s' (%s)\n",
+                GNUNET_h2s (&issue->denom_hash),
+                TALER_amount2s (&value));
+    TALER_amount_ntoh (&value,
+                       &issue->fee_withdraw);
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Withdraw fee is %s\n",
+                TALER_amount2s (&value));
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Start time is %s\n",
+                GNUNET_STRINGS_absolute_time_to_string
+                  (GNUNET_TIME_absolute_ntoh (issue->start)));
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Expire deposit time is %s\n",
+                GNUNET_STRINGS_absolute_time_to_string
+                  (GNUNET_TIME_absolute_ntoh (issue->expire_deposit)));
+  }
+  i = GNUNET_new (struct TALER_DenominationKeyValidityPS);
+  *i = *issue;
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap_put (denominations,
+                                                    &issue->denom_hash,
+                                                    i,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  return GNUNET_OK;
+}
+
+
+/**
+ * Obtain information about a @a denom_pub.
+ *
+ * @param dh hash of the denomination public key to look up
+ * @param[out] issue set to detailed information about @a denom_pub, NULL if not found, must
+ *                 NOT be freed by caller
+ * @return transaction status code
+ */
+enum GNUNET_DB_QueryStatus
+get_denomination_info_by_hash (const struct GNUNET_HashCode *dh,
+                               const struct
+                               TALER_DenominationKeyValidityPS **issue)
+{
+  const struct TALER_DenominationKeyValidityPS *i;
+  enum GNUNET_DB_QueryStatus qs;
+
+  if (NULL == denominations)
+  {
+    denominations = GNUNET_CONTAINER_multihashmap_create (256,
+                                                          GNUNET_NO);
+    qs = adb->select_denomination_info (adb->cls,
+                                        asession,
+                                        &master_pub,
+                                        &add_denomination,
+                                        NULL);
+    if (0 > qs)
+    {
+      *issue = NULL;
+      return qs;
+    }
+  }
+  i = GNUNET_CONTAINER_multihashmap_get (denominations,
+                                         dh);
+  if (NULL != i)
+  {
+    /* cache hit */
+    *issue = i;
+    return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
+  }
+  /* maybe database changed since we last iterated, give it one more shot */
+  qs = adb->select_denomination_info (adb->cls,
+                                      asession,
+                                      &master_pub,
+                                      &add_denomination,
+                                      NULL);
+  if (qs <= 0)
+  {
+    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "Denomination %s not found\n",
+                  TALER_B2S (dh));
+    return qs;
+  }
+  i = GNUNET_CONTAINER_multihashmap_get (denominations,
+                                         dh);
+  if (NULL != i)
+  {
+    /* cache hit */
+    *issue = i;
+    return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
+  }
+  /* We found more keys, but not the denomination we are looking for :-( */
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Denomination %s not found\n",
+              TALER_B2S (dh));
+  return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS;
+}
+
+
+/**
+ * Obtain information about a @a denom_pub.
+ *
+ * @param denom_pub key to look up
+ * @param[out] issue set to detailed information about @a denom_pub, NULL if not found, must
+ *                 NOT be freed by caller
+ * @param[out] dh set to the hash of @a denom_pub, may be NULL
+ * @return transaction status code
+ */
+enum GNUNET_DB_QueryStatus
+get_denomination_info (const struct TALER_DenominationPublicKey *denom_pub,
+                       const struct
+                       TALER_DenominationKeyValidityPS **issue,
+                       struct GNUNET_HashCode *dh)
+{
+  struct GNUNET_HashCode hc;
+
+  if (NULL == dh)
+    dh = &hc;
+  GNUNET_CRYPTO_rsa_public_key_hash (denom_pub->rsa_public_key,
+                                     dh);
+  return get_denomination_info_by_hash (dh,
+                                        issue);
+}
+
+
+/**
+ * Perform the given @a analysis within a transaction scope.
+ * Commit on success.
+ *
+ * @param analysis analysis to run
+ * @param analysis_cls closure for @a analysis
+ * @return #GNUNET_OK if @a analysis succeessfully committed,
+ *         #GNUNET_NO if we had an error on commit (retry may help)
+ *         #GNUNET_SYSERR on hard errors
+ */
+int
+transact (Analysis analysis,
+          void *analysis_cls)
+{
+  int ret;
+  enum GNUNET_DB_QueryStatus qs;
+
+  ret = adb->start (adb->cls,
+                    asession);
+  if (GNUNET_OK != ret)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  edb->preflight (edb->cls,
+                  esession);
+  ret = edb->start (edb->cls,
+                    esession,
+                    "auditor");
+  if (GNUNET_OK != ret)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  qs = analysis (analysis_cls);
+  if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
+  {
+    qs = edb->commit (edb->cls,
+                      esession);
+    if (0 > qs)
+    {
+      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Exchange DB commit failed, rolling back transaction\n");
+      adb->rollback (adb->cls,
+                     asession);
+    }
+    else
+    {
+      qs = adb->commit (adb->cls,
+                        asession);
+      if (0 > qs)
+      {
+        GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
+        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                    "Auditor DB commit failed!\n");
+      }
+    }
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Processing failed (or no changes), rolling back transaction\n");
+    adb->rollback (adb->cls,
+                   asession);
+    edb->rollback (edb->cls,
+                   esession);
+  }
+  switch (qs)
+  {
+  case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
+    return GNUNET_OK;
+  case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
+    return GNUNET_OK;
+  case GNUNET_DB_STATUS_SOFT_ERROR:
+    return GNUNET_NO;
+  case GNUNET_DB_STATUS_HARD_ERROR:
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Initialize DB sessions and run the analysis.
+ *
+ * @param ana analysis to run
+ * @param ana_cls closure for @ana
+ * @return #GNUNET_OK on success
+ */
+int
+setup_sessions_and_run (Analysis ana,
+                        void *ana_cls)
+{
+  esession = edb->get_session (edb->cls);
+  if (NULL == esession)
+  {
+    fprintf (stderr,
+             "Failed to initialize exchange session.\n");
+    return GNUNET_SYSERR;
+  }
+  asession = adb->get_session (adb->cls);
+  if (NULL == asession)
+  {
+    fprintf (stderr,
+             "Failed to initialize auditor session.\n");
+    return GNUNET_SYSERR;
+  }
+
+  GNUNET_break (GNUNET_SYSERR !=
+                transact (ana,
+                          ana_cls));
+  return GNUNET_OK;
+}
+
+
+/**
+ * Test if the given @a mpub matches the #master_pub.
+ * If so, set "found" to GNUNET_YES.
+ *
+ * @param cls a `int *` pointing to "found"
+ * @param mpub exchange master public key to compare
+ * @param exchange_url URL of the exchange (ignored)
+ */
+static void
+test_master_present (void *cls,
+                     const struct TALER_MasterPublicKeyP *mpub,
+                     const char *exchange_url)
+{
+  int *found = cls;
+
+  (void) exchange_url;
+  if (0 == GNUNET_memcmp (mpub,
+                          &master_pub))
+    *found = GNUNET_YES;
+}
+
+
+int
+setup_globals (const struct GNUNET_CONFIGURATION_Handle *c)
+{
+  int found;
+  struct TALER_AUDITORDB_Session *as;
+
+  cfg = c;
+  start_time = GNUNET_TIME_absolute_get ();
+  if (0 == GNUNET_is_zero (&master_pub))
+  {
+    /* -m option not given, try configuration */
+    char *master_public_key_str;
+
+    if (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_string (cfg,
+                                               "exchange",
+                                               "MASTER_PUBLIC_KEY",
+                                               &master_public_key_str))
+    {
+      fprintf (stderr,
+               "Pass option -m or set it in the configuration!\n");
+      GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                                 "exchange",
+                                 "MASTER_PUBLIC_KEY");
+      return GNUNET_SYSERR;
+    }
+    if (GNUNET_OK !=
+        GNUNET_CRYPTO_eddsa_public_key_from_string (master_public_key_str,
+                                                    strlen (
+                                                      master_public_key_str),
+                                                    &master_pub.eddsa_pub))
+    {
+      fprintf (stderr,
+               "Invalid master public key given in configuration file.");
+      GNUNET_free (master_public_key_str);
+      return GNUNET_SYSERR;
+    }
+    GNUNET_free (master_public_key_str);
+  } /* end of -m not given */
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Taler auditor running for exchange master public key %s\n",
+              TALER_B2S (&master_pub));
+
+  if (GNUNET_OK !=
+      TALER_config_get_currency (cfg,
+                                 &currency))
+  {
+    return GNUNET_SYSERR;
+  }
+  {
+    if (GNUNET_OK !=
+        TALER_config_get_amount (cfg,
+                                 "taler",
+                                 "CURRENCY_ROUND_UNIT",
+                                 &currency_round_unit))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Invalid or missing amount in `TALER' under `CURRENCY_ROUND_UNIT'\n");
+      return GNUNET_SYSERR;
+    }
+  }
+  if (NULL ==
+      (edb = TALER_EXCHANGEDB_plugin_load (cfg)))
+  {
+    fprintf (stderr,
+             "Failed to initialize exchange database plugin.\n");
+    return GNUNET_SYSERR;
+  }
+  if (NULL ==
+      (adb = TALER_AUDITORDB_plugin_load (cfg)))
+  {
+    fprintf (stderr,
+             "Failed to initialize auditor database plugin.\n");
+    TALER_EXCHANGEDB_plugin_unload (edb);
+    return GNUNET_SYSERR;
+  }
+  found = GNUNET_NO;
+  as = adb->get_session (adb->cls);
+  if (NULL == as)
+  {
+    fprintf (stderr,
+             "Failed to start session with auditor database.\n");
+    TALER_AUDITORDB_plugin_unload (adb);
+    TALER_EXCHANGEDB_plugin_unload (edb);
+    return GNUNET_SYSERR;
+  }
+  (void) adb->list_exchanges (adb->cls,
+                              as,
+                              &test_master_present,
+                              &found);
+  if (GNUNET_NO == found)
+  {
+    fprintf (stderr,
+             "Exchange's master public key `%s' not known to auditor DB. Did you forget to run `taler-auditor-exchange`?\n",
+             GNUNET_p2s (&master_pub.eddsa_pub));
+    TALER_AUDITORDB_plugin_unload (adb);
+    TALER_EXCHANGEDB_plugin_unload (edb);
+    return GNUNET_SYSERR;
+  }
+  if (restart)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Full audit restart requested, dropping old audit data.\n");
+    GNUNET_break (GNUNET_OK ==
+                  adb->drop_tables (adb->cls,
+                                    GNUNET_NO));
+    TALER_AUDITORDB_plugin_unload (adb);
+    if (NULL ==
+        (adb = TALER_AUDITORDB_plugin_load (cfg)))
+    {
+      fprintf (stderr,
+               "Failed to initialize auditor database plugin after drop.\n");
+      TALER_EXCHANGEDB_plugin_unload (edb);
+      return GNUNET_SYSERR;
+    }
+    GNUNET_break (GNUNET_OK ==
+                  adb->create_tables (adb->cls));
+  }
+
+  return GNUNET_OK;
+}
+
+
+void
+finish_report (json_t *report)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Audit complete\n");
+  TALER_AUDITORDB_plugin_unload (adb);
+  adb = NULL;
+  TALER_EXCHANGEDB_plugin_unload (edb);
+  edb = NULL;
+  json_dumpf (report,
+              stdout,
+              JSON_INDENT (2));
+  json_decref (report);
+}

src/auditor/report-lib.h

@@ -0,0 +1,188 @@
+/*
+  This file is part of TALER
+  Copyright (C) 2016-2020 Taler Systems SA
+
+  TALER is free software; you can redistribute it and/or modify it under the
+  terms of the GNU Affero Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU Affero Public License for more details.
+
+  You should have received a copy of the GNU Affero Public License along with
+  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file auditor/report-lib.h
+ * @brief helper library to facilitate generation of audit reports
+ * @author Christian Grothoff
+ */
+#ifndef REPORT_LIB_H
+#define REPORT_LIB_H
+
+#include <gnunet/gnunet_util_lib.h>
+#include "taler_auditordb_plugin.h"
+#include "taler_exchangedb_lib.h"
+#include "taler_json_lib.h"
+#include "taler_bank_service.h"
+#include "taler_signatures.h"
+
+
+/**
+ * Command-line option "-r": restart audit from scratch
+ */
+extern int restart;
+
+/**
+ * Handle to access the exchange's database.
+ */
+extern struct TALER_EXCHANGEDB_Plugin *edb;
+
+/**
+ * Which currency are we doing the audit for?
+ */
+extern char *currency;
+
+/**
+ * How many fractional digits does the currency use?
+ */
+extern struct TALER_Amount currency_round_unit;
+
+/**
+ * Our configuration.
+ */
+extern const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
+ * Our session with the #edb.
+ */
+extern struct TALER_EXCHANGEDB_Session *esession;
+
+/**
+ * Handle to access the auditor's database.
+ */
+extern struct TALER_AUDITORDB_Plugin *adb;
+
+/**
+ * Our session with the #adb.
+ */
+extern struct TALER_AUDITORDB_Session *asession;
+
+/**
+ * Master public key of the exchange to audit.
+ */
+extern struct TALER_MasterPublicKeyP master_pub;
+
+/**
+ * At what time did the auditor process start?
+ */
+extern struct GNUNET_TIME_Absolute start_time;
+
+
+/**
+ * Convert absolute time to human-readable JSON string.
+ *
+ * @param at time to convert
+ * @return human-readable string representing the time
+ */
+json_t *
+json_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO at);
+
+
+/**
+ * Convert absolute time to human-readable JSON string.
+ *
+ * @param at time to convert
+ * @return human-readable string representing the time
+ */
+json_t *
+json_from_time_abs (struct GNUNET_TIME_Absolute at);
+
+
+/**
+ * Add @a object to the report @a array.  Fail hard if this fails.
+ *
+ * @param array report array to append @a object to
+ * @param object object to append, should be check that it is not NULL
+ */
+void
+report (json_t *array,
+        json_t *object);
+
+
+/**
+ * Obtain information about a @a denom_pub.
+ *
+ * @param dh hash of the denomination public key to look up
+ * @param[out] issue set to detailed information about @a denom_pub, NULL if not found, must
+ *                 NOT be freed by caller
+ * @return transaction status code
+ */
+enum GNUNET_DB_QueryStatus
+get_denomination_info_by_hash (
+  const struct GNUNET_HashCode *dh,
+  const struct TALER_DenominationKeyValidityPS **issue);
+
+
+/**
+ * Obtain information about a @a denom_pub.
+ *
+ * @param denom_pub key to look up
+ * @param[out] issue set to detailed information about @a denom_pub, NULL if not found, must
+ *                 NOT be freed by caller
+ * @param[out] dh set to the hash of @a denom_pub, may be NULL
+ * @return transaction status code
+ */
+enum GNUNET_DB_QueryStatus
+get_denomination_info (
+  const struct TALER_DenominationPublicKey *denom_pub,
+  const struct TALER_DenominationKeyValidityPS **issue,
+  struct GNUNET_HashCode *dh);
+
+/**
+ * Type of an analysis function.  Each analysis function runs in
+ * its own transaction scope and must thus be internally consistent.
+ *
+ * @param cls closure
+ * @return transaction status code
+ */
+typedef enum GNUNET_DB_QueryStatus
+(*Analysis)(void *cls);
+
+
+/**
+ * Perform the given @a analysis within a transaction scope.
+ * Commit on success.
+ *
+ * @param analysis analysis to run
+ * @param analysis_cls closure for @a analysis
+ * @return #GNUNET_OK if @a analysis succeessfully committed,
+ *         #GNUNET_NO if we had an error on commit (retry may help)
+ *         #GNUNET_SYSERR on hard errors
+ */
+int
+transact (Analysis analysis,
+          void *analysis_cls);
+
+
+/**
+ * Initialize DB sessions and run the analysis.
+ *
+ * @param ana analysis to run
+ * @param ana_cls closure for @ana
+ * @return #GNUNET_OK on success
+ */
+int
+setup_sessions_and_run (Analysis ana,
+                        void *ana_cls);
+
+
+int
+setup_globals (const struct GNUNET_CONFIGURATION_Handle *c);
+
+
+void
+finish_report (json_t *report);
+
+#endif

src/auditor/taler-auditor-deposits.c

@@ -0,0 +1,360 @@
+/*
+  This file is part of TALER
+  Copyright (C) 2016-2020 Taler Systems SA
+
+  TALER is free software; you can redistribute it and/or modify it under the
+  terms of the GNU Affero Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU Affero Public License for more details.
+
+  You should have received a copy of the GNU Affero Public License along with
+  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file auditor/taler-auditor-deposits.c
+ * @brief audits an exchange database for deposit confirmation consistency
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include <gnunet/gnunet_util_lib.h>
+#include "taler_auditordb_plugin.h"
+#include "taler_exchangedb_lib.h"
+#include "taler_json_lib.h"
+#include "taler_bank_service.h"
+#include "taler_signatures.h"
+#include "report-lib.h"
+
+
+/**
+ * Return value from main().
+ */
+static int global_ret;
+
+/**
+ * Array of reports about missing deposit confirmations.
+ */
+static json_t *report_deposit_confirmation_inconsistencies;
+
+/**
+ * Total number of deposit confirmations that we did not get.
+ */
+static json_int_t number_missed_deposit_confirmations;
+
+/**
+ * Total amount involved in deposit confirmations that we did not get.
+ */
+static struct TALER_Amount total_missed_deposit_confirmations;
+
+
+/* *************************** Analysis of deposit-confirmations ********** */
+
+/**
+ * Closure for #test_dc.
+ */
+struct DepositConfirmationContext
+{
+
+  /**
+   * How many deposit confirmations did we NOT find in the #edb?
+   */
+  unsigned long long missed_count;
+
+  /**
+   * What is the total amount missing?
+   */
+  struct TALER_Amount missed_amount;
+
+  /**
+   * Lowest SerialID of the first coin we missed? (This is where we
+   * should resume next time).
+   */
+  uint64_t first_missed_coin_serial;
+
+  /**
+   * Lowest SerialID of the first coin we missed? (This is where we
+   * should resume next time).
+   */
+  uint64_t last_seen_coin_serial;
+
+  /**
+   * Success or failure of (exchange) database operations within
+   * #test_dc.
+   */
+  enum GNUNET_DB_QueryStatus qs;
+
+};
+
+
+/**
+ * Given a deposit confirmation from #adb, check that it is also
+ * in #edb.  Update the deposit confirmation context accordingly.
+ *
+ * @param cls our `struct DepositConfirmationContext`
+ * @param serial_id row of the @a dc in the database
+ * @param dc the deposit confirmation we know
+ */
+static void
+test_dc (void *cls,
+         uint64_t serial_id,
+         const struct TALER_AUDITORDB_DepositConfirmation *dc)
+{
+  struct DepositConfirmationContext *dcc = cls;
+  enum GNUNET_DB_QueryStatus qs;
+  struct TALER_EXCHANGEDB_Deposit dep;
+
+  dcc->last_seen_coin_serial = serial_id;
+  memset (&dep,
+          0,
+          sizeof (dep));
+  dep.coin.coin_pub = dc->coin_pub;
+  dep.h_contract_terms = dc->h_contract_terms;
+  dep.merchant_pub = dc->merchant;
+  dep.h_wire = dc->h_wire;
+  dep.refund_deadline = dc->refund_deadline;
+
+  qs = edb->have_deposit (edb->cls,
+                          esession,
+                          &dep,
+                          GNUNET_NO /* do not check refund deadline */);
+  if (qs > 0)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Found deposit %s in exchange database\n",
+                GNUNET_h2s (&dc->h_contract_terms));
+    return;   /* found, all good */
+  }
+  if (qs < 0)
+  {
+    GNUNET_break (0); /* DB error, complain */
+    dcc->qs = qs;
+    return;
+  }
+  /* deposit confirmation missing! report! */
+  report (report_deposit_confirmation_inconsistencies,
+          json_pack ("{s:o, s:o, s:I, s:o}",
+                     "timestamp",
+                     json_from_time_abs (dc->timestamp),
+                     "amount",
+                     TALER_JSON_from_amount (&dc->amount_without_fee),
+                     "rowid",
+                     (json_int_t) serial_id,
+                     "account",
+                     GNUNET_JSON_from_data_auto (&dc->h_wire)));
+  dcc->first_missed_coin_serial = GNUNET_MIN (dcc->first_missed_coin_serial,
+                                              serial_id);
+  dcc->missed_count++;
+  GNUNET_assert (GNUNET_OK ==
+                 TALER_amount_add (&dcc->missed_amount,
+                                   &dcc->missed_amount,
+                                   &dc->amount_without_fee));
+}
+
+
+/**
+ * Check that the deposit-confirmations that were reported to
+ * us by merchants are also in the exchange's database.
+ *
+ * @param cls closure
+ * @return transaction status code
+ */
+static enum GNUNET_DB_QueryStatus
+analyze_deposit_confirmations (void *cls)
+{
+  struct TALER_AUDITORDB_ProgressPointDepositConfirmation ppdc;
+  struct DepositConfirmationContext dcc;
+  enum GNUNET_DB_QueryStatus qs;
+  enum GNUNET_DB_QueryStatus qsx;
+  enum GNUNET_DB_QueryStatus qsp;
+
+  (void) cls;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Analyzing deposit confirmations\n");
+  ppdc.last_deposit_confirmation_serial_id = 0;
+  qsp = adb->get_auditor_progress_deposit_confirmation (adb->cls,
+                                                        asession,
+                                                        &master_pub,
+                                                        &ppdc);
+  if (0 > qsp)
+  {
+    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp);
+    return qsp;
+  }
+  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
+                _ (
+                  "First analysis using this auditor, starting audit from scratch\n"));
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _ ("Resuming deposit confirmation audit at %llu\n"),
+                (unsigned long long) ppdc.last_deposit_confirmation_serial_id);
+  }
+
+  /* setup 'cc' */
+  GNUNET_assert (GNUNET_OK ==
+                 TALER_amount_get_zero (currency,
+                                        &dcc.missed_amount));
+  dcc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
+  dcc.missed_count = 0LLU;
+  dcc.first_missed_coin_serial = UINT64_MAX;
+  qsx = adb->get_deposit_confirmations (adb->cls,
+                                        asession,
+                                        &master_pub,
+                                        ppdc.last_deposit_confirmation_serial_id,
+                                        &test_dc,
+                                        &dcc);
+  if (0 > qsx)
+  {
+    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx);
+    return qsx;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Analyzed %d deposit confirmations (above serial ID %llu)\n",
+              (int) qsx,
+              (unsigned long long) ppdc.last_deposit_confirmation_serial_id);
+  if (0 > dcc.qs)
+  {
+    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == dcc.qs);
+    return dcc.qs;
+  }
+  if (UINT64_MAX == dcc.first_missed_coin_serial)
+    ppdc.last_deposit_confirmation_serial_id = dcc.last_seen_coin_serial;
+  else
+    ppdc.last_deposit_confirmation_serial_id = dcc.first_missed_coin_serial - 1;
+
+  /* sync 'cc' back to disk */
+  if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp)
+    qs = adb->update_auditor_progress_deposit_confirmation (adb->cls,
+                                                            asession,
+                                                            &master_pub,
+                                                            &ppdc);
+  else
+    qs = adb->insert_auditor_progress_deposit_confirmation (adb->cls,
+                                                            asession,
+                                                            &master_pub,
+                                                            &ppdc);
+  if (0 >= qs)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Failed to update auditor DB, not recording progress\n");
+    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
+    return qs;
+  }
+  number_missed_deposit_confirmations = (json_int_t) dcc.missed_count;
+  total_missed_deposit_confirmations = dcc.missed_amount;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _ ("Concluded deposit confirmation audit step at %llu\n"),
+              (unsigned long long) ppdc.last_deposit_confirmation_serial_id);
+  return qs;
+}
+
+
+/**
+ * Main function that will be run.
+ *
+ * @param cls closure
+ * @param args remaining command-line arguments
+ * @param cfgfile name of the configuration file used (for saving, can be NULL!)
+ * @param c configuration
+ */
+static void
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
+     const struct GNUNET_CONFIGURATION_Handle *c)
+{
+  json_t *report;
+
+  (void) cls;
+  (void) args;
+  (void) cfgfile;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Launching auditor\n");
+  if (GNUNET_OK !=
+      setup_globals (c))
+  {
+    global_ret = 1;
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Starting audit\n");
+  GNUNET_assert (NULL !=
+                 (report_deposit_confirmation_inconsistencies = json_array ()));
+  if (GNUNET_OK !=
+      setup_sessions_and_run (&analyze_deposit_confirmations,
+                              NULL))
+  {
+    global_ret = 1;
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Audit complete\n");
+  report = json_pack ("{s:o, s:o, s:I}",
+                      "deposit_confirmation_inconsistencies",
+                      report_deposit_confirmation_inconsistencies,
+                      "missing_deposit_confirmation_count",
+                      (json_int_t) number_missed_deposit_confirmations,
+                      "missing_deposit_confirmation_total",
+                      TALER_JSON_from_amount (
+                        &total_missed_deposit_confirmations)
+                      );
+  GNUNET_break (NULL != report);
+  finish_report (report);
+}
+
+
+/**
+ * The main function of the database initialization tool.
+ * Used to initialize the Taler Exchange's database.
+ *
+ * @param argc number of arguments from the command line
+ * @param argv command line arguments
+ * @return 0 ok, 1 on error
+ */
+int
+main (int argc,
+      char *const *argv)
+{
+  const struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_base32_auto ('m',
+                                      "exchange-key",
+                                      "KEY",
+                                      "public key of the exchange (Crockford base32 encoded)",
+                                      &master_pub),
+    GNUNET_GETOPT_option_flag ('r',
+                               "restart",
+                               "restart audit from the beginning (required on first run)",
+                               &restart),
+    GNUNET_GETOPT_option_timetravel ('T',
+                                     "timetravel"),
+    GNUNET_GETOPT_OPTION_END
+  };
+
+  /* force linker to link against libtalerutil; if we do
+     not do this, the linker may "optimize" libtalerutil
+     away and skip #TALER_OS_init(), which we do need */
+  (void) TALER_project_data_default ();
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_log_setup ("taler-auditor-deposits",
+                                   "MESSAGE",
+                                   NULL));
+  if (GNUNET_OK !=
+      GNUNET_PROGRAM_run (argc,
+                          argv,
+                          "taler-auditor-deposits",
+                          "Audit Taler exchange database for deposit confirmation consistency",
+                          options,
+                          &run,
+                          NULL))
+    return 1;
+  return global_ret;
+}
+
+
+/* end of taler-auditor-deposits.c */

src/auditor/taler-auditor.c

@@ -18,6 +18,13 @@
  * @brief audits an exchange database.
  * @author Christian Grothoff
  *
+ * README-FIRST:
+ *
+ * This code is being split up into
+ * taler-auditor-{aggregation,coins,deposits,reserves}.  It is still here as a
+ * reference, but this file should be obsolete once the split has been
+ * completed. DO NOT EDIT THIS FILE ANYMORE! -CG
+ *
  * NOTE:
  * - This auditor does not verify that 'reserves_in' actually matches
  *   the wire transfers from the bank. This needs to be checked separately!