Merge branch 'master' into ar

contrib/gana

@@ -1 +1 @@
-Subproject commit b7320181c5e0d95c6f2e2a9e5c53dce0bc1a35a8
+Subproject commit b123140349c3e3b300878d2e35cea1553c9a381d

contrib/uncrustify.cfg

@@ -28,7 +28,7 @@ ls_code_width=true
 pos_arith=lead
 
 # Fully parenthesize boolean exprs
-mod_full_paren_if_bool=true
+mod_full_paren_if_bool=false
 
 # Braces should be on their own line
 nl_fdef_brace=add

src/bank-lib/fakebank.c

@@ -480,11 +480,14 @@ lp_trigger (struct LongPoller *lp,
   MHD_resume_connection (lp->conn);
   GNUNET_free (lp);
   h->mhd_again = true;
-  if (NULL != h->mhd_task)
+  if (-1 != h->lp_event)
-    GNUNET_SCHEDULER_cancel (h->mhd_task);
+  {
-  h->mhd_task =
+    if (NULL != h->mhd_task)
-    GNUNET_SCHEDULER_add_now (&run_mhd,
+      GNUNET_SCHEDULER_cancel (h->mhd_task);
-                              h);
+    h->mhd_task =
+      GNUNET_SCHEDULER_add_now (&run_mhd,
+                                h);
+  }
 }
 
 
@@ -2413,6 +2416,7 @@ schedule_httpd (struct TALER_FAKEBANK_Handle *h)
   MHD_UNSIGNED_LONG_LONG timeout;
   struct GNUNET_TIME_Relative tv;
 
+  GNUNET_assert (-1 != h->mhd_fd);
   haveto = MHD_get_timeout (h->mhd_bank,
                             &timeout);
   if (MHD_YES == haveto)
@@ -2450,6 +2454,7 @@ schedule_httpd (struct TALER_FAKEBANK_Handle *h)
   MHD_UNSIGNED_LONG_LONG timeout;
   struct GNUNET_TIME_Relative tv;
 
+  GNUNET_assert (-1 == h->lp_event);
   FD_ZERO (&rs);
   FD_ZERO (&ws);
   FD_ZERO (&es);
@@ -2521,6 +2526,7 @@ run_mhd (void *cls)
     h->mhd_again = false;
     MHD_run (h->mhd_bank);
   }
+  GNUNET_assert (-1 == h->lp_event);
   schedule_httpd (h);
 }
 
@@ -2554,6 +2560,7 @@ TALER_FAKEBANK_start2 (uint16_t port,
   GNUNET_assert (strlen (currency) < TALER_CURRENCY_LEN);
   h = GNUNET_new (struct TALER_FAKEBANK_Handle);
   h->lp_event = -1;
+  h->mhd_fd = -1;
   h->port = port;
   h->ram_limit = ram_limit;
   h->serial_counter = 0;

src/curl/curl.c

@@ -30,14 +30,6 @@
 #endif
 
 
-/**
- * Add the @a body as POST data to the easy handle in @a ctx.
- *
- * @param[in,out] ctx a request context (updated)
- * @param eh easy handle to use
- * @param body JSON body to add to @e ctx
- * @return #GNUNET_OK on success #GNUNET_SYSERR on failure
- */
 enum GNUNET_GenericReturnValue
 TALER_curl_easy_post (struct TALER_CURL_PostContext *ctx,
                       CURL *eh,
@@ -101,11 +93,6 @@ TALER_curl_easy_post (struct TALER_CURL_PostContext *ctx,
 }
 
 
-/**
- * Free the data in @a ctx.
- *
- * @param[in] ctx a request context (updated)
- */
 void
 TALER_curl_easy_post_finished (struct TALER_CURL_PostContext *ctx)
 {

src/exchange-tools/.gitignore

@@ -1,2 +1,3 @@
 taler-exchange-offline
 taler-auditor-offline
+taler-crypto-worker

src/exchange-tools/Makefile.am

@@ -15,7 +15,8 @@ endif
 bin_PROGRAMS = \
   taler-auditor-offline \
   taler-exchange-offline \
-  taler-exchange-dbinit
+  taler-exchange-dbinit \
+  taler-crypto-worker
 
 taler_exchange_offline_SOURCES = \
   taler-exchange-offline.c
@@ -59,6 +60,20 @@ taler_exchange_dbinit_CPPFLAGS = \
   -I$(top_srcdir)/src/pq/ \
   $(POSTGRESQL_CPPFLAGS)
 
+taler_crypto_worker_SOURCES = \
+  taler-crypto-worker.c
+taler_crypto_worker_LDADD = \
+  $(top_builddir)/src/util/libtalerutil.la \
+  $(top_builddir)/src/json/libtalerjson.la \
+  -lgnunetutil \
+  -lgnunetjson \
+  -ljansson \
+  -lpthread \
+  $(LIBGCRYPT_LIBS) \
+  $(XLIB)
+
+
+
 
 # Testcases
 

src/exchange-tools/taler-crypto-worker.c

@@ -25,7 +25,6 @@
 #include "taler_error_codes.h"
 #include "taler_json_lib.h"
 #include "taler_signatures.h"
-#include "secmod_common.h"
 
 
 /**
@@ -138,80 +137,24 @@ run (void *cls,
                   "sent response\n");
       continue;
     }
-    if (0 == strcmp ("setup_refresh_planchet", op))
+    if (0 == strcmp ("eddsa_sign",
+                     op))
     {
-      struct TALER_DenominationPublicKey denom_pub;
+      struct GNUNET_CRYPTO_EddsaSignature sig;
-      struct TALER_Amount fee_withdraw;
+      struct GNUNET_CRYPTO_EccSignaturePurpose *msg;
-      struct TALER_Amount value;
+      struct GNUNET_CRYPTO_EddsaPrivateKey priv;
-      struct TALER_ReservePublicKeyP reserve_pub;
+      size_t msg_size;
-      struct TALER_ReservePublicKeyP reserve_priv;
-      uint32_t coin_index;
       json_t *resp;
-      struct GNUNET_JSON_Specification eddsa_verify_spec[] = {
+      struct GNUNET_JSON_Specification eddsa_sign_spec[] = {
-        TALER_JSON_spec_denom_pub ("denom_pub",
+        GNUNET_JSON_spec_fixed_auto ("priv",
-                                   &denom_pub),
+                                     &priv),
-        TALER_JSON_spec_amount_any ("fee_withdraw",
+        GNUNET_JSON_spec_varsize ("msg",
-                                    &fee_withdraw),
+                                  (void **) &msg,
-        TALER_JSON_spec_amount_any ("value",
+                                  &msg_size),
-                                    &value),
-        GNUNET_JSON_spec_fixed_auto ("reserve_pub",
-                                     &reserve_pub),
-        GNUNET_JSON_spec_fixed_auto ("reserve_priv",
-                                     &reserve_priv),
-        GNUNET_JSON_spec_uint32 ("coin_index",
-                                 &coin_index),
-        GNUNET_JSON_spec_end ()
-      };
-      struct TALER_CoinSpendPublicKeyP coin_pub;
-      struct TALER_PlanchetSecretsP ps;
-
-      if (GNUNET_OK !=
-          GNUNET_JSON_parse (args,
-                             eddsa_verify_spec,
-                             NULL,
-                             NULL))
-      {
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "malformed op args\n");
-        global_ret = 1;
-        return;
-      }
-#if FIXME_FLORIAN
-      TALER_planchet_setup_refresh (&transfer_secret,
-                                    coin_num_salt,
-                                    &ps);
-#endif
-      GNUNET_CRYPTO_eddsa_key_get_public (&ps.coin_priv.eddsa_priv,
-                                          &coin_pub.eddsa_pub);
-
-      resp = GNUNET_JSON_PACK (
-        GNUNET_JSON_pack_data_auto ("coin_priv", &ps.coin_priv),
-        GNUNET_JSON_pack_data_auto ("coin_pub", &coin_pub),
-        GNUNET_JSON_pack_data_auto ("blinding_key", &ps.blinding_key)
-        );
-      json_dumpf (resp, stdout, JSON_COMPACT);
-      printf ("\n");
-      fflush (stdout);
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  "sent response\n");
-      continue;
-    }
-    if (0 == strcmp (op, "create_planchet"))
-    {
-      struct TALER_TransferSecretP transfer_secret;
-      uint32_t coin_num_salt;
-      struct TALER_PlanchetSecretsP ps;
-      struct TALER_CoinSpendPublicKeyP coin_pub;
-      json_t *resp;
-      struct GNUNET_JSON_Specification eddsa_verify_spec[] = {
-        GNUNET_JSON_spec_fixed_auto ("transfer_secret",
-                                     &transfer_secret),
-        GNUNET_JSON_spec_uint32 ("coin_index",
-                                 &coin_num_salt),
         GNUNET_JSON_spec_end ()
       };
       if (GNUNET_OK != GNUNET_JSON_parse (args,
-                                          eddsa_verify_spec,
+                                          eddsa_sign_spec,
                                           NULL,
                                           NULL))
       {
@@ -220,8 +163,50 @@ run (void *cls,
         global_ret = 1;
         return;
       }
+      GNUNET_CRYPTO_eddsa_sign_ (
+        &priv,
+        msg,
+        &sig
+        );
+      resp = GNUNET_JSON_PACK (
+        GNUNET_JSON_pack_data_auto ("sig", &sig)
+        );
+      json_dumpf (resp, stdout, JSON_COMPACT);
+      printf ("\n");
+      fflush (stdout);
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "sent response\n");
+      continue;
+    }
+    if (0 == strcmp ("setup_refresh_planchet", op))
+    {
+      struct TALER_TransferSecretP transfer_secret;
+      uint32_t coin_index;
+      json_t *resp;
+      struct GNUNET_JSON_Specification setup_refresh_planchet_spec[] = {
+        GNUNET_JSON_spec_uint32 ("coin_index",
+                                 &coin_index),
+        GNUNET_JSON_spec_fixed_auto ("transfer_secret",
+                                     &transfer_secret),
+        GNUNET_JSON_spec_end ()
+      };
+      struct TALER_CoinSpendPublicKeyP coin_pub;
+      struct TALER_PlanchetSecretsP ps;
+
+      if (GNUNET_OK !=
+          GNUNET_JSON_parse (args,
+                             setup_refresh_planchet_spec,
+                             NULL,
+                             NULL))
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                    "malformed op args\n");
+        global_ret = 1;
+        return;
+      }
       TALER_planchet_setup_refresh (&transfer_secret,
-                                    coin_num_salt, &ps);
+                                    coin_index,
+                                    &ps);
       GNUNET_CRYPTO_eddsa_key_get_public (&ps.coin_priv.eddsa_priv,
                                           &coin_pub.eddsa_pub);
 

src/exchange/Makefile.am

@@ -97,6 +97,7 @@ taler_exchange_httpd_SOURCES = \
   taler-exchange-httpd_management_wire_disable.c \
   taler-exchange-httpd_management_wire_fees.c \
   taler-exchange-httpd_melt.c taler-exchange-httpd_melt.h \
+  taler-exchange-httpd_metrics.c taler-exchange-httpd_metrics.h \
   taler-exchange-httpd_mhd.c taler-exchange-httpd_mhd.h \
   taler-exchange-httpd_recoup.c taler-exchange-httpd_recoup.h \
   taler-exchange-httpd_refreshes_reveal.c taler-exchange-httpd_refreshes_reveal.h \

src/exchange/taler-exchange-httpd.c

@@ -39,6 +39,7 @@
 #include "taler-exchange-httpd_link.h"
 #include "taler-exchange-httpd_management.h"
 #include "taler-exchange-httpd_melt.h"
+#include "taler-exchange-httpd_metrics.h"
 #include "taler-exchange-httpd_mhd.h"
 #include "taler-exchange-httpd_recoup.h"
 #include "taler-exchange-httpd_refreshes_reveal.h"
@@ -57,6 +58,11 @@
  */
 #define UNIX_BACKLOG 50
 
+/**
+ * Above what request latency do we start to log?
+ */
+#define WARN_LATENCY GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_MILLISECONDS, 500)
 
 /**
  * Are clients allowed to request /keys for times other than the
@@ -382,6 +388,18 @@ handle_mhd_completion_callback (void *cls,
   /* Sanity-check that we didn't leave any transactions hanging */
   GNUNET_break (GNUNET_OK ==
                 TEH_plugin->preflight (TEH_plugin->cls));
+  {
+    struct GNUNET_TIME_Relative latency;
+
+    latency = GNUNET_TIME_absolute_get_duration (rc->start_time);
+    if (latency.rel_value_us >
+        WARN_LATENCY.rel_value_us)
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "Request for `%s' took %s\n",
+                  rc->url,
+                  GNUNET_STRINGS_relative_time_to_string (latency,
+                                                          GNUNET_YES));
+  }
   GNUNET_free (rc);
   *con_cls = NULL;
   GNUNET_async_scope_restore (&old_scope);
@@ -849,6 +867,12 @@ handle_mhd_request (void *cls,
       .method = MHD_HTTP_METHOD_GET,
       .handler.get = &handler_seed
     },
+    /* Performance metrics */
+    {
+      .url = "metrics",
+      .method = MHD_HTTP_METHOD_GET,
+      .handler.get = &TEH_handler_metrics
+    },
     /* Terms of service */
     {
       .url = "terms",
@@ -980,6 +1004,7 @@ handle_mhd_request (void *cls,
 
     /* We're in a new async scope! */
     rc = *con_cls = GNUNET_new (struct TEH_RequestContext);
+    rc->start_time = GNUNET_TIME_absolute_get ();
     GNUNET_async_scope_fresh (&rc->async_scope_id);
     TEH_check_invariants ();
     rc->url = url;
@@ -996,6 +1021,36 @@ handle_mhd_request (void *cls,
                   "illegal incoming correlation ID\n");
       correlation_id = NULL;
     }
+
+    /* Check if upload is in bounds */
+    if (0 == strcasecmp (method,
+                         MHD_HTTP_METHOD_POST))
+    {
+      const char *cl;
+
+      /* Maybe check for maximum upload size
+         and refuse requests if they are just too big. */
+      cl = MHD_lookup_connection_value (connection,
+                                        MHD_HEADER_KIND,
+                                        MHD_HTTP_HEADER_CONTENT_LENGTH);
+      if (NULL != cl)
+      {
+        unsigned long long cv;
+        char dummy;
+
+        if (1 != sscanf (cl,
+                         "%llu%c",
+                         &cv,
+                         &dummy))
+        {
+          /* Not valid HTTP request, just close connection. */
+          GNUNET_break_op (0);
+          return MHD_NO;
+        }
+        if (cv > TALER_MHD_REQUEST_BUFFER_MAX)
+          return TALER_MHD_reply_request_too_large (connection);
+      }
+    }
   }
 
   GNUNET_async_scope_enter (&rc->async_scope_id,

src/exchange/taler-exchange-httpd.h

@@ -218,6 +218,11 @@ struct TEH_RequestContext
    */
   struct GNUNET_AsyncScopeId async_scope_id;
 
+  /**
+   * When was this request started?
+   */
+  struct GNUNET_TIME_Absolute start_time;
+
   /**
    * Opaque parsing context.
    */

src/exchange/taler-exchange-httpd_auditors.c

@@ -216,6 +216,7 @@ TEH_handler_auditors (
     return MHD_YES;   /* failure */
   ret = TEH_DB_run_transaction (connection,
                                 "add auditor denom sig",
+                                TEH_MT_OTHER,
                                 &res,
                                 &add_auditor_denom_sig,
                                 &awc);

src/exchange/taler-exchange-httpd_db.c

@@ -1,6 +1,6 @@
 /*
   This file is part of TALER
-  Copyright (C) 2014-2017 Taler Systems SA
+  Copyright (C) 2014-2017, 2021 Taler Systems SA
 
   TALER is free software; you can redistribute it and/or modify it under the
   terms of the GNU General Public License as published by the Free Software
@@ -24,9 +24,60 @@
 #include <gnunet/gnunet_json_lib.h>
 #include "taler_json_lib.h"
 #include "taler_mhd_lib.h"
+#include "taler_exchangedb_lib.h"
+#include "taler-exchange-httpd_db.h"
 #include "taler-exchange-httpd_responses.h"
 
 
+/**
+ * Send a response for a failed request.  The transaction history of the given
+ * coin demonstrates that the @a residual value of the coin is below the @a
+ * requested contribution of the coin for the operation.  Thus, the exchange
+ * refuses the operation.
+ *
+ * @param connection the connection to send the response to
+ * @param coin_pub public key of the coin
+ * @param coin_value original value of the coin
+ * @param tl transaction history for the coin
+ * @param requested how much this coin was supposed to contribute, including fee
+ * @param residual remaining value of the coin (after subtracting @a tl)
+ * @return a MHD result code
+ */
+static MHD_RESULT
+reply_insufficient_funds (
+  struct MHD_Connection *connection,
+  const struct TALER_CoinSpendPublicKeyP *coin_pub,
+  const struct TALER_Amount *coin_value,
+  struct TALER_EXCHANGEDB_TransactionList *tl,
+  const struct TALER_Amount *requested,
+  const struct TALER_Amount *residual)
+{
+  json_t *history;
+
+  history = TEH_RESPONSE_compile_transaction_history (coin_pub,
+                                                      tl);
+  if (NULL == history)
+    return TALER_MHD_reply_with_error (connection,
+                                       MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                       TALER_EC_EXCHANGE_GENERIC_HISTORY_DB_ERROR_INSUFFICIENT_FUNDS,
+                                       NULL);
+  return TALER_MHD_REPLY_JSON_PACK (
+    connection,
+    MHD_HTTP_CONFLICT,
+    TALER_JSON_pack_ec (TALER_EC_EXCHANGE_GENERIC_INSUFFICIENT_FUNDS),
+    GNUNET_JSON_pack_data_auto ("coin_pub",
+                                coin_pub),
+    TALER_JSON_pack_amount ("original_value",
+                            coin_value),
+    TALER_JSON_pack_amount ("residual_value",
+                            residual),
+    TALER_JSON_pack_amount ("requested_value",
+                            requested),
+    GNUNET_JSON_pack_array_steal ("history",
+                                  history));
+}
+
+
 /**
  * How often should we retry a transaction before giving up
  * (for transactions resulting in serialization/dead locks only).
@@ -98,6 +149,8 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin,
     // FIXME: why do we even return the transaction
     // history here!? This is a coin with multiple
     // associated denominations, after all...
+    // => this is probably the wrong call, as this
+    // is NOT about insufficient funds!
     *mhd_ret
       = TEH_RESPONSE_reply_coin_insufficient_funds (
           connection,
@@ -112,24 +165,217 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin,
 
 
 /**
- * Run a database transaction for @a connection.
+ * Called when we actually know that the balance (was) insufficient.
- * Starts a transaction and calls @a cb.  Upon success,
+ * Re-does the check (slowly) to compute the full error message for
- * attempts to commit the transaction.  Upon soft failures,
+ * the client.
- * retries @a cb a few times.  Upon hard or persistent soft
- * errors, generates an error message for @a connection.
  *
- * @param connection MHD connection to run @a cb for, can be NULL
+ * @param connection HTTP connection to report hard errors on
- * @param name name of the transaction (for debugging)
+ * @param coin_pub coin to analyze
- * @param[out] mhd_ret set to MHD response code, if transaction failed;
+ * @param coin_value total value of the original coin (by denomination)
- *             NULL if we are not running with a @a connection and thus
+ * @param op_cost cost of the current operation (for error reporting)
- *             must not queue MHD replies
+ * @param check_recoup should we include recoup transactions in the check
- * @param cb callback implementing transaction logic
+ * @param zombie_required additional requirement that the coin must
- * @param cb_cls closure for @a cb, must be read-only!
+ *          be a zombie coin, or also hard failure
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+ * @param[out] mhd_ret set to response status code, on hard error only
+ * @return transaction status
  */
+static enum GNUNET_DB_QueryStatus
+check_coin_balance (struct MHD_Connection *connection,
+                    const struct TALER_CoinSpendPublicKeyP *coin_pub,
+                    const struct TALER_Amount *coin_value,
+                    const struct TALER_Amount *op_cost,
+                    bool check_recoup,
+                    bool zombie_required,
+                    MHD_RESULT *mhd_ret)
+{
+  struct TALER_EXCHANGEDB_TransactionList *tl;
+  struct TALER_Amount spent;
+  enum GNUNET_DB_QueryStatus qs;
+
+  /* Start with zero cost, as we already added this melt transaction
+     to the DB, so we will see it again during the queries below. */
+  GNUNET_assert (GNUNET_OK ==
+                 TALER_amount_set_zero (TEH_currency,
+                                        &spent));
+
+  /* get historic transaction costs of this coin, including recoups as
+     we might be a zombie coin */
+  qs = TEH_plugin->get_coin_transactions (TEH_plugin->cls,
+                                          coin_pub,
+                                          check_recoup,
+                                          &tl);
+  if (0 > qs)
+  {
+    if (GNUNET_DB_STATUS_HARD_ERROR == qs)
+      *mhd_ret = TALER_MHD_reply_with_error (connection,
+                                             MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                             TALER_EC_GENERIC_DB_FETCH_FAILED,
+                                             "coin transaction history");
+    return qs;
+  }
+  if (zombie_required)
+  {
+    /* The denomination key is only usable for a melt if this is a true
+       zombie coin, i.e. it was refreshed and the resulting fresh coin was
+       then recouped. Check that this is truly the case. */
+    for (struct TALER_EXCHANGEDB_TransactionList *tp = tl;
+         NULL != tp;
+         tp = tp->next)
+    {
+      if (TALER_EXCHANGEDB_TT_OLD_COIN_RECOUP == tp->type)
+      {
+        zombie_required = false; /* clear flag: was satisfied! */
+        break;
+      }
+    }
+    if (zombie_required)
+    {
+      /* zombie status not satisfied */
+      GNUNET_break_op (0);
+      TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
+                                              tl);
+      *mhd_ret = TALER_MHD_reply_with_error (connection,
+                                             MHD_HTTP_BAD_REQUEST,
+                                             TALER_EC_EXCHANGE_MELT_COIN_EXPIRED_NO_ZOMBIE,
+                                             NULL);
+      return GNUNET_DB_STATUS_HARD_ERROR;
+    }
+  }
+  if (GNUNET_OK !=
+      TALER_EXCHANGEDB_calculate_transaction_list_totals (tl,
+                                                          &spent,
+                                                          &spent))
+  {
+    GNUNET_break (0);
+    TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
+                                            tl);
+    *mhd_ret = TALER_MHD_reply_with_error (connection,
+                                           MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                           TALER_EC_EXCHANGE_GENERIC_COIN_HISTORY_COMPUTATION_FAILED,
+                                           NULL);
+    return GNUNET_DB_STATUS_HARD_ERROR;
+  }
+
+  /* Refuse to refresh when the coin's value is insufficient
+     for the cost of all transactions. */
+  if (0 > TALER_amount_cmp (coin_value,
+                            &spent))
+  {
+    struct TALER_Amount coin_residual;
+    struct TALER_Amount spent_already;
+
+    /* First subtract the melt cost from 'spent' to
+       compute the total amount already spent of the coin */
+    GNUNET_assert (0 <=
+                   TALER_amount_subtract (&spent_already,
+                                          &spent,
+                                          op_cost));
+    /* The residual coin value is the original coin value minus
+       what we have spent (before the melt) */
+    GNUNET_assert (0 <=
+                   TALER_amount_subtract (&coin_residual,
+                                          coin_value,
+                                          &spent_already));
+    *mhd_ret = reply_insufficient_funds (
+      connection,
+      coin_pub,
+      coin_value,
+      tl,
+      op_cost,
+      &coin_residual);
+    TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
+                                            tl);
+    return GNUNET_DB_STATUS_HARD_ERROR;
+  }
+
+  /* This should not happen: The coin has sufficient funds
+     after all!?!? */
+  TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
+                                          tl);
+  return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
+}
+
+
+enum GNUNET_DB_QueryStatus
+TEH_check_coin_balance (struct MHD_Connection *connection,
+                        const struct TALER_CoinSpendPublicKeyP *coin_pub,
+                        const struct TALER_Amount *coin_value,
+                        const struct TALER_Amount *op_cost,
+                        bool check_recoup,
+                        bool zombie_required,
+                        MHD_RESULT *mhd_ret)
+{
+  bool balance_ok = false;
+  bool zombie_ok = false;
+  enum GNUNET_DB_QueryStatus qs;
+
+  qs = TEH_plugin->do_check_coin_balance (TEH_plugin->cls,
+                                          coin_pub,
+                                          coin_value,
+                                          check_recoup,
+                                          zombie_required,
+                                          &balance_ok,
+                                          &zombie_ok);
+  switch (qs)
+  {
+  case GNUNET_DB_STATUS_HARD_ERROR:
+    *mhd_ret = TALER_MHD_reply_with_error (
+      connection,
+      MHD_HTTP_INTERNAL_SERVER_ERROR,
+      TALER_EC_GENERIC_DB_FETCH_FAILED,
+      "check_coin_balance");
+    return qs;
+  case GNUNET_DB_STATUS_SOFT_ERROR:
+    return qs;
+  case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
+    GNUNET_break (0);
+    *mhd_ret = TALER_MHD_reply_with_error (
+      connection,
+      MHD_HTTP_INTERNAL_SERVER_ERROR,
+      TALER_EC_GENERIC_DB_FETCH_FAILED,
+      "check_coin_balance");
+    return GNUNET_DB_STATUS_HARD_ERROR;
+  case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
+    /* handled below */
+    break;
+  }
+  if (! zombie_ok)
+  {
+    GNUNET_break_op (0);
+    *mhd_ret = TALER_MHD_reply_with_error (
+      connection,
+      MHD_HTTP_BAD_REQUEST,
+      TALER_EC_EXCHANGE_MELT_COIN_EXPIRED_NO_ZOMBIE,
+      NULL);
+    return GNUNET_DB_STATUS_HARD_ERROR;
+  }
+  if (balance_ok)
+    return qs;
+  /* balance is not OK, do expensive call to compute full error message */
+  qs = check_coin_balance (connection,
+                           coin_pub,
+                           coin_value,
+                           op_cost,
+                           check_recoup,
+                           zombie_required,
+                           mhd_ret);
+  if (qs < 0)
+    return qs; /* we expected to fail (same check as before!) */
+  GNUNET_break (0); /* stored procedure and individual statements
+                       disagree, should be impossible! */
+  *mhd_ret = TALER_MHD_reply_with_error (
+    connection,
+    MHD_HTTP_INTERNAL_SERVER_ERROR,
+    TALER_EC_GENERIC_DB_INVARIANT_FAILURE,
+    "stored procedure disagrees with full coin transaction history fetch");
+  return GNUNET_DB_STATUS_HARD_ERROR;
+}
+
+
 enum GNUNET_GenericReturnValue
 TEH_DB_run_transaction (struct MHD_Connection *connection,
                         const char *name,
+                        enum TEH_MetricType mt,
                         MHD_RESULT *mhd_ret,
                         TEH_DB_TransactionCallback cb,
                         void *cb_cls)
@@ -147,6 +393,8 @@ TEH_DB_run_transaction (struct MHD_Connection *connection,
                                              NULL);
     return GNUNET_SYSERR;
   }
+  GNUNET_assert (mt < TEH_MT_COUNT);
+  TEH_METRICS_num_requests[mt]++;
   for (unsigned int retries = 0;
        retries < MAX_TRANSACTION_COMMIT_RETRIES;
        retries++)
@@ -173,21 +421,27 @@ TEH_DB_run_transaction (struct MHD_Connection *connection,
     if (GNUNET_DB_STATUS_HARD_ERROR == qs)
       return GNUNET_SYSERR;
     if (0 <= qs)
-      qs = TEH_plugin->commit (TEH_plugin->cls);
-    if (GNUNET_DB_STATUS_HARD_ERROR == qs)
     {
-      if (NULL != mhd_ret)
+      qs = TEH_plugin->commit (TEH_plugin->cls);
-        *mhd_ret = TALER_MHD_reply_with_error (connection,
+      if (GNUNET_DB_STATUS_HARD_ERROR == qs)
-                                               MHD_HTTP_INTERNAL_SERVER_ERROR,
+      {
-                                               TALER_EC_GENERIC_DB_COMMIT_FAILED,
+        TEH_plugin->rollback (TEH_plugin->cls);
-                                               NULL);
+        if (NULL != mhd_ret)
-      return GNUNET_SYSERR;
+          *mhd_ret = TALER_MHD_reply_with_error (connection,
+                                                 MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                                 TALER_EC_GENERIC_DB_COMMIT_FAILED,
+                                                 NULL);
+        return GNUNET_SYSERR;
+      }
+      if (0 > qs)
+        TEH_plugin->rollback (TEH_plugin->cls);
     }
     /* make sure callback did not violate invariants! */
     GNUNET_assert ( (NULL == mhd_ret) ||
                     (-1 == (int) *mhd_ret) );
     if (0 <= qs)
       return GNUNET_OK;
+    TEH_METRICS_num_conflict[mt]++;
   }
   TALER_LOG_ERROR ("Transaction `%s' commit failed %u times\n",
                    name,

src/exchange/taler-exchange-httpd_db.h

@@ -23,6 +23,7 @@
 
 #include <microhttpd.h>
 #include "taler_exchangedb_plugin.h"
+#include "taler-exchange-httpd_metrics.h"
 #include <gnunet/gnunet_mhd_compat.h>
 
 
@@ -40,6 +41,35 @@ TEH_make_coin_known (const struct TALER_CoinPublicInfo *coin,
                      MHD_RESULT *mhd_ret);
 
 
+/**
+ * Check that a coin has an adequate balance so that we can
+ * commit the current transaction. If the balance is
+ * insufficient for all transactions associated with the
+ * coin, return a hard error.
+ *
+ * We first do a "fast" check using a stored procedure, and
+ * only obtain the "full" data on failure (for performance).
+ *
+ * @param connection HTTP connection to report hard errors on
+ * @param coin_pub coin to analyze
+ * @param coin_value total value of the original coin (by denomination)
+ * @param op_cost cost of the current operation (for error reporting)
+ * @param check_recoup should we include recoup transactions in the check
+ * @param zombie_required additional requirement that the coin must
+ *          be a zombie coin, or also hard failure
+ * @param[out] mhd_ret set to response status code, on hard error only
+ * @return transaction status
+ */
+enum GNUNET_DB_QueryStatus
+TEH_check_coin_balance (struct MHD_Connection *connection,
+                        const struct TALER_CoinSpendPublicKeyP *coin_pub,
+                        const struct TALER_Amount *coin_value,
+                        const struct TALER_Amount *op_cost,
+                        bool check_recoup,
+                        bool zombie_required,
+                        MHD_RESULT *mhd_ret);
+
+
 /**
  * Function implementing a database transaction.  Runs the transaction
  * logic; IF it returns a non-error code, the transaction logic MUST
@@ -69,6 +99,7 @@ typedef enum GNUNET_DB_QueryStatus
  *
  * @param connection MHD connection to run @a cb for, can be NULL
  * @param name name of the transaction (for debugging)
+ * @param mt type of the requests, for metric generation
  * @param[out] mhd_ret set to MHD response code, if transaction failed (returned #GNUNET_SYSERR);
  *             NULL if we are not running with a @a connection and thus
  *             must not queue MHD replies
@@ -79,6 +110,7 @@ typedef enum GNUNET_DB_QueryStatus
 enum GNUNET_GenericReturnValue
 TEH_DB_run_transaction (struct MHD_Connection *connection,
                         const char *name,
+                        enum TEH_MetricType mt,
                         MHD_RESULT *mhd_ret,
                         TEH_DB_TransactionCallback cb,
                         void *cb_cls);

src/exchange/taler-exchange-httpd_deposit.c

@@ -162,120 +162,85 @@ deposit_transaction (void *cls,
   enum GNUNET_DB_QueryStatus qs;
   struct TALER_Amount deposit_fee;
 
-  /* make sure coin is 'known' in database */
+  /* begin optimistically: assume this is a new deposit */
-  qs = TEH_make_coin_known (&deposit->coin,
+  qs = TEH_plugin->insert_deposit (TEH_plugin->cls,
-                            connection,
+                                   dc->exchange_timestamp,
-                            mhd_ret);
+                                   deposit);
-  if (qs < 0)
-    return qs;
-
-  /* Check for idempotency: did we get this request before? */
-  qs = TEH_plugin->have_deposit (TEH_plugin->cls,
-                                 deposit,
-                                 &deposit_fee,
-                                 &dc->exchange_timestamp);
   if (qs < 0)
   {
-    if (GNUNET_DB_STATUS_HARD_ERROR == qs)
+    if (GNUNET_DB_STATUS_SOFT_ERROR == qs)
+      return qs;
+    TALER_LOG_WARNING ("Failed to store /deposit information in database\n");
+    *mhd_ret = TALER_MHD_reply_with_error (connection,
+                                           MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                           TALER_EC_GENERIC_DB_STORE_FAILED,
+                                           NULL);
+    return qs;
+  }
+  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+  {
+    /* Check for idempotency: did we get this request before? */
+    qs = TEH_plugin->have_deposit (TEH_plugin->cls,
+                                   deposit,
+                                   &deposit_fee,
+                                   &dc->exchange_timestamp);
+    if (qs < 0)
     {
+      if (GNUNET_DB_STATUS_SOFT_ERROR == qs)
+        return qs;
       *mhd_ret = TALER_MHD_reply_with_error (connection,
                                              MHD_HTTP_INTERNAL_SERVER_ERROR,
                                              TALER_EC_GENERIC_DB_FETCH_FAILED,
                                              "have_deposit");
       return GNUNET_DB_STATUS_HARD_ERROR;
     }
-    return qs;
+    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
-  }
-  if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
-  {
-    struct TALER_Amount amount_without_fee;
-
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                "/deposit replay, accepting again!\n");
-    GNUNET_assert (0 <=
-                   TALER_amount_subtract (&amount_without_fee,
-                                          &deposit->amount_with_fee,
-                                          &deposit_fee));
-    *mhd_ret = reply_deposit_success (connection,
-                                      &deposit->coin.coin_pub,
-                                      &dc->h_wire,
-                                      NULL /* h_extensions! */,
-                                      &deposit->h_contract_terms,
-                                      dc->exchange_timestamp,
-                                      deposit->refund_deadline,
-                                      deposit->wire_deadline,
-                                      &deposit->merchant_pub,
-                                      &amount_without_fee);
-    return GNUNET_DB_STATUS_HARD_ERROR;
-  }
-
-  /* Start with fee for THIS transaction */
-  spent = deposit->amount_with_fee;
-  /* add cost of all previous transactions; skip RECOUP as revoked
-     denominations are not eligible for deposit, and if we are the old coin
-     pub of a revoked coin (aka a zombie), then ONLY refresh is allowed. */
-  {
-    struct TALER_EXCHANGEDB_TransactionList *tl;
-
-    qs = TEH_plugin->get_coin_transactions (TEH_plugin->cls,
-                                            &deposit->coin.coin_pub,
-                                            GNUNET_NO,
-                                            &tl);
-    if (0 > qs)
     {
-      if (GNUNET_DB_STATUS_HARD_ERROR == qs)
+      /* Conflict on insert, but record does not exist?
-        *mhd_ret = TALER_MHD_reply_with_error (
+         That makes no sense. */
-          connection,
+      GNUNET_break (0);
-          MHD_HTTP_INTERNAL_SERVER_ERROR,
-          TALER_EC_GENERIC_DB_FETCH_FAILED,
-          NULL);
-      return qs;
-    }
-    if (GNUNET_OK !=
-        TALER_EXCHANGEDB_calculate_transaction_list_totals (tl,
-                                                            &spent, /* starting offset */
-                                                            &spent /* result */))
-    {
-      TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
-                                              tl);
-      *mhd_ret = TALER_MHD_reply_with_error (
-        connection,
-        MHD_HTTP_INTERNAL_SERVER_ERROR,
-        TALER_EC_GENERIC_DB_INVARIANT_FAILURE,
-        NULL);
       return GNUNET_DB_STATUS_HARD_ERROR;
     }
-    /* Check that cost of all transactions (including the current one) is
+
-       smaller (or equal) than the value of the coin. */
-    if (0 < TALER_amount_cmp (&spent,
-                              &dc->value))
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+      struct TALER_Amount amount_without_fee;
-                  "Deposited coin has insufficient funds left!\n");
+
-      *mhd_ret = TEH_RESPONSE_reply_coin_insufficient_funds (connection,
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                                                             TALER_EC_EXCHANGE_DEPOSIT_INSUFFICIENT_FUNDS,
+                  "/deposit replay, accepting again!\n");
-                                                             &deposit->coin.
+      GNUNET_assert (0 <=
-                                                             coin_pub,
+                     TALER_amount_subtract (&amount_without_fee,
-                                                             tl);
+                                            &deposit->amount_with_fee,
-      TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
+                                            &deposit_fee));
-                                              tl);
+      *mhd_ret = reply_deposit_success (connection,
+                                        &deposit->coin.coin_pub,
+                                        &dc->h_wire,
+                                        NULL /* h_extensions! */,
+                                        &deposit->h_contract_terms,
+                                        dc->exchange_timestamp,
+                                        deposit->refund_deadline,
+                                        deposit->wire_deadline,
+                                        &deposit->merchant_pub,
+                                        &amount_without_fee);
+      /* Note: we return "hard error" to ensure the wrapper
+         does not retry the transaction, and to also not generate
+         a "fresh" response (as we would on "success") */
       return GNUNET_DB_STATUS_HARD_ERROR;
     }
-    TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
-                                            tl);
   }
-  qs = TEH_plugin->insert_deposit (TEH_plugin->cls,
+
-                                   dc->exchange_timestamp,
+  /* Start with zero cost, as we already added this melt transaction
-                                   deposit);
+     to the DB, so we will see it again during the queries below. */
-  if (GNUNET_DB_STATUS_HARD_ERROR == qs)
+  GNUNET_assert (GNUNET_OK ==
-  {
+                 TALER_amount_set_zero (TEH_currency,
-    TALER_LOG_WARNING ("Failed to store /deposit information in database\n");
+                                        &spent));
-    *mhd_ret = TALER_MHD_reply_with_error (connection,
+
-                                           MHD_HTTP_INTERNAL_SERVER_ERROR,
+  return TEH_check_coin_balance (connection,
-                                           TALER_EC_GENERIC_DB_STORE_FAILED,
+                                 &deposit->coin.coin_pub,
-                                           NULL);
+                                 &dc->value,
-  }
+                                 &deposit->amount_with_fee,
-  return qs;
+                                 false, /* no need for recoup */
+                                 false, /* no need for zombie */
+                                 mhd_ret);
 }
 
 
@@ -490,6 +455,31 @@ TEH_handler_deposit (struct MHD_Connection *connection,
                                        NULL);
   }
 
+  if (GNUNET_SYSERR ==
+      TEH_plugin->preflight (TEH_plugin->cls))
+  {
+    GNUNET_break (0);
+    return TALER_MHD_reply_with_error (connection,
+                                       MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                       TALER_EC_GENERIC_DB_START_FAILED,
+                                       "preflight failure");
+  }
+
+  {
+    MHD_RESULT mhd_ret = MHD_NO;
+    enum GNUNET_DB_QueryStatus qs;
+
+    /* make sure coin is 'known' in database */
+    qs = TEH_make_coin_known (&deposit.coin,
+                              connection,
+                              &mhd_ret);
+    /* no transaction => no serialization failures should be possible */
+    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs);
+    if (qs < 0)
+      return mhd_ret;
+  }
+
+
   /* execute transaction */
   {
     MHD_RESULT mhd_ret;
@@ -497,6 +487,7 @@ TEH_handler_deposit (struct MHD_Connection *connection,
     if (GNUNET_OK !=
         TEH_DB_run_transaction (connection,
                                 "execute deposit",
+                                TEH_MT_DEPOSIT,
                                 &mhd_ret,
                                 &deposit_transaction,
                                 &dc))

src/exchange/taler-exchange-httpd_deposits_get.c

@@ -246,6 +246,7 @@ handle_track_transaction_request (
   if (GNUNET_OK !=
       TEH_DB_run_transaction (connection,
                               "handle deposits GET",
+                              TEH_MT_OTHER,
                               &mhd_ret,
                               &deposits_get_transaction,
                               &ctx))

src/exchange/taler-exchange-httpd_keys.c

@@ -322,9 +322,10 @@ struct TEH_KeyStateHandle
   struct GNUNET_TIME_Absolute reload_time;
 
   /**
-   * When is the next key invalid and we expect to have a different reply?
+   * What is the period at which we rotate keys
+   * (signing or denomination keys)?
    */
-  struct GNUNET_TIME_Absolute next_reload;
+  struct GNUNET_TIME_Relative rekey_frequency;
 
   /**
    * When does our online signing key expire and we
@@ -1370,9 +1371,9 @@ auditor_denom_cb (
 struct SignKeyCtx
 {
   /**
-   * When does the next signing key expire. Updated.
+   * What is the current rotation frequency for signing keys. Updated.
    */
-  struct GNUNET_TIME_Absolute next_sk_expire;
+  struct GNUNET_TIME_Relative min_sk_frequency;
 
   /**
    * JSON array of signing keys (being created).
@@ -1399,10 +1400,14 @@ add_sign_key_cb (void *cls,
   struct SigningKey *sk = value;
 
   (void) pid;
-  ctx->next_sk_expire =
+  if (GNUNET_TIME_absolute_is_future (sk->meta.expire_sign))
-    GNUNET_TIME_absolute_min (ctx->next_sk_expire,
+  {
-                              sk->meta.expire_sign);
+    ctx->min_sk_frequency =
-
+      GNUNET_TIME_relative_min (ctx->min_sk_frequency,
+                                GNUNET_TIME_absolute_get_difference (
+                                  sk->meta.start,
+                                  sk->meta.expire_sign));
+  }
   GNUNET_assert (
     0 ==
     json_array_append_new (
@@ -1438,9 +1443,10 @@ struct DenomKeyCtx
   json_t *recoup;
 
   /**
-   * When does the next denomination key expire. Updated.
+   * What is the minimum key rotation frequency of
+   * valid denomination keys?
    */
-  struct GNUNET_TIME_Absolute next_dk_expire;
+  struct GNUNET_TIME_Relative min_dk_frequency;
 
 };
 
@@ -1475,9 +1481,14 @@ add_denom_key_cb (void *cls,
   }
   else
   {
-    dkc->next_dk_expire =
+    if (GNUNET_TIME_absolute_is_future (dk->meta.start))
-      GNUNET_TIME_absolute_min (dkc->next_dk_expire,
+    {
-                                dk->meta.expire_withdraw);
+      dkc->min_dk_frequency =
+        GNUNET_TIME_relative_min (dkc->min_dk_frequency,
+                                  GNUNET_TIME_absolute_get_difference (
+                                    dk->meta.start,
+                                    dk->meta.expire_withdraw));
+    }
     (void) GNUNET_CONTAINER_heap_insert (dkc->heap,
                                          dk,
                                          dk->meta.start.abs_value_us);
@@ -1546,7 +1557,7 @@ get_date_string (struct GNUNET_TIME_Absolute at,
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
-setup_general_response_headers (const struct TEH_KeyStateHandle *ksh,
+setup_general_response_headers (struct TEH_KeyStateHandle *ksh,
                                 struct MHD_Response *response)
 {
   char dat[128];
@@ -1562,13 +1573,14 @@ setup_general_response_headers (const struct TEH_KeyStateHandle *ksh,
                 MHD_add_response_header (response,
                                          MHD_HTTP_HEADER_LAST_MODIFIED,
                                          dat));
-  if (0 != ksh->next_reload.abs_value_us)
+  if (! GNUNET_TIME_relative_is_zero (ksh->rekey_frequency))
   {
+    struct GNUNET_TIME_Relative r;
     struct GNUNET_TIME_Absolute m;
 
-    m = GNUNET_TIME_relative_to_absolute (TEH_max_keys_caching);
+    r = GNUNET_TIME_relative_min (TEH_max_keys_caching,
-    m = GNUNET_TIME_absolute_min (m,
+                                  ksh->rekey_frequency);
-                                  ksh->next_reload);
+    m = GNUNET_TIME_relative_to_absolute (r);
     get_date_string (m,
                      dat);
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
@@ -1578,6 +1590,9 @@ setup_general_response_headers (const struct TEH_KeyStateHandle *ksh,
                   MHD_add_response_header (response,
                                            MHD_HTTP_HEADER_EXPIRES,
                                            dat));
+    ksh->signature_expires
+      = GNUNET_TIME_absolute_min (m,
+                                  ksh->signature_expires);
   }
   return GNUNET_OK;
 }
@@ -1759,7 +1774,7 @@ finish_keys_response (struct TEH_KeyStateHandle *ksh)
 
   sctx.signkeys = json_array ();
   GNUNET_assert (NULL != sctx.signkeys);
-  sctx.next_sk_expire = GNUNET_TIME_UNIT_FOREVER_ABS;
+  sctx.min_sk_frequency = GNUNET_TIME_UNIT_FOREVER_REL;
   GNUNET_CONTAINER_multipeermap_iterate (ksh->signkey_map,
                                          &add_sign_key_cb,
                                          &sctx);
@@ -1770,15 +1785,15 @@ finish_keys_response (struct TEH_KeyStateHandle *ksh)
     struct DenomKeyCtx dkc = {
       .recoup = recoup,
       .heap = heap,
-      .next_dk_expire = GNUNET_TIME_UNIT_FOREVER_ABS,
+      .min_dk_frequency = GNUNET_TIME_UNIT_FOREVER_REL,
     };
 
     GNUNET_CONTAINER_multihashmap_iterate (ksh->denomkey_map,
                                            &add_denom_key_cb,
                                            &dkc);
-    ksh->next_reload
+    ksh->rekey_frequency
-      = GNUNET_TIME_absolute_min (dkc.next_dk_expire,
+      = GNUNET_TIME_relative_min (dkc.min_dk_frequency,
-                                  sctx.next_sk_expire);
+                                  sctx.min_sk_frequency);
   }
   denoms = json_array ();
   GNUNET_assert (NULL != denoms);
@@ -1935,6 +1950,8 @@ build_key_state (struct HelperState *hs,
   ksh->auditors = json_array ();
   GNUNET_assert (NULL != ksh->auditors);
   /* NOTE: fetches master-signed signkeys, but ALSO those that were revoked! */
+  GNUNET_break (GNUNET_OK ==
+                TEH_plugin->preflight (TEH_plugin->cls));
   qs = TEH_plugin->iterate_denominations (TEH_plugin->cls,
                                           &denomination_info_cb,
                                           ksh);

src/exchange/taler-exchange-httpd_kyc-check.c

@@ -351,6 +351,7 @@ TEH_handler_kyc_check (
   (void) GNUNET_TIME_round_abs (&now);
   ret = TEH_DB_run_transaction (rc->connection,
                                 "kyc check",
+                                TEH_MT_OTHER,
                                 &res,
                                 &kyc_check,
                                 kyp);

src/exchange/taler-exchange-httpd_kyc-proof.c

@@ -678,6 +678,7 @@ TEH_handler_kyc_proof (
 
     ret = TEH_DB_run_transaction (kpc->rc->connection,
                                   "check proof kyc",
+                                  TEH_MT_OTHER,
                                   &res,
                                   &persist_kyc_ok,
                                   kpc);

src/exchange/taler-exchange-httpd_kyc-wallet.c

@@ -140,6 +140,7 @@ TEH_handler_kyc_wallet (
       0);
   ret = TEH_DB_run_transaction (rc->connection,
                                 "check wallet kyc",
+                                TEH_MT_OTHER,
                                 &res,
                                 &wallet_kyc_check,
                                 &krc);

src/exchange/taler-exchange-httpd_link.c

@@ -193,6 +193,7 @@ TEH_handler_link (struct TEH_RequestContext *rc,
   if (GNUNET_OK !=
       TEH_DB_run_transaction (rc->connection,
                               "run link",
+                              TEH_MT_OTHER,
                               &mhd_ret,
                               &link_transaction,
                               &ctx))

src/exchange/taler-exchange-httpd_management_auditors.c

@@ -187,6 +187,7 @@ TEH_handler_management_auditors (
 
   ret = TEH_DB_run_transaction (connection,
                                 "add auditor",
+                                TEH_MT_OTHER,
                                 &res,
                                 &add_auditor,
                                 &aac);

src/exchange/taler-exchange-httpd_management_auditors_AP_disable.c

@@ -176,6 +176,7 @@ TEH_handler_management_auditors_AP_disable (
 
   ret = TEH_DB_run_transaction (connection,
                                 "del auditor",
+                                TEH_MT_OTHER,
                                 &res,
                                 &del_auditor,
                                 &dac);

src/exchange/taler-exchange-httpd_management_post_keys.c

@@ -367,6 +367,8 @@ TEH_handler_management_post_keys (
       TALER_EC_GENERIC_PARAMETER_MALFORMED,
       "array expected for denom_sigs and signkey_sigs");
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Received /management/keys\n");
   akc.nd_sigs = json_array_size (denom_sigs);
   akc.d_sigs = GNUNET_new_array (akc.nd_sigs,
                                  struct DenomSig);
@@ -404,6 +406,8 @@ TEH_handler_management_post_keys (
   {
     GNUNET_free (akc.d_sigs);
     GNUNET_JSON_parse_free (spec);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failure to handle /management/keys\n");
     return ret;
   }
   akc.ns_sigs = json_array_size (signkey_sigs);
@@ -440,6 +444,8 @@ TEH_handler_management_post_keys (
   }
   if (! ok)
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failure to handle /management/keys\n");
     GNUNET_free (akc.d_sigs);
     GNUNET_free (akc.s_sigs);
     GNUNET_JSON_parse_free (spec);
@@ -454,6 +460,7 @@ TEH_handler_management_post_keys (
 
     res = TEH_DB_run_transaction (connection,
                                   "add keys",
+                                  TEH_MT_OTHER,
                                   &ret,
                                   &add_keys,
                                   &akc);

src/exchange/taler-exchange-httpd_management_wire_disable.c

@@ -177,6 +177,7 @@ TEH_handler_management_post_wire_disable (
 
     res = TEH_DB_run_transaction (connection,
                                   "del wire",
+                                  TEH_MT_OTHER,
                                   &ret,
                                   &del_wire,
                                   &awc);

src/exchange/taler-exchange-httpd_management_wire_enable.c

@@ -212,6 +212,7 @@ TEH_handler_management_post_wire (
 
     res = TEH_DB_run_transaction (connection,
                                   "add wire",
+                                  TEH_MT_OTHER,
                                   &ret,
                                   &add_wire,
                                   &awc);

src/exchange/taler-exchange-httpd_management_wire_fees.c

@@ -221,6 +221,7 @@ TEH_handler_management_post_wire_fees (
 
     res = TEH_DB_run_transaction (connection,
                                   "add wire fee",
+                                  TEH_MT_OTHER,
                                   &ret,
                                   &add_fee,
                                   &afc);

src/exchange/taler-exchange-httpd_melt.c

@@ -33,56 +33,6 @@
 #include "taler_exchangedb_lib.h"
 
 
-/**
- * Send a response for a failed "melt" request.  The
- * transaction history of the given coin demonstrates that the
- * @a residual value of the coin is below the @a requested
- * contribution of the coin for the melt.  Thus, the exchange
- * refuses the melt operation.
- *
- * @param connection the connection to send the response to
- * @param coin_pub public key of the coin
- * @param coin_value original value of the coin
- * @param tl transaction history for the coin
- * @param requested how much this coin was supposed to contribute, including fee
- * @param residual remaining value of the coin (after subtracting @a tl)
- * @return a MHD result code
- */
-static MHD_RESULT
-reply_melt_insufficient_funds (
-  struct MHD_Connection *connection,
-  const struct TALER_CoinSpendPublicKeyP *coin_pub,
-  const struct TALER_Amount *coin_value,
-  struct TALER_EXCHANGEDB_TransactionList *tl,
-  const struct TALER_Amount *requested,
-  const struct TALER_Amount *residual)
-{
-  json_t *history;
-
-  history = TEH_RESPONSE_compile_transaction_history (coin_pub,
-                                                      tl);
-  if (NULL == history)
-    return TALER_MHD_reply_with_error (connection,
-                                       MHD_HTTP_INTERNAL_SERVER_ERROR,
-                                       TALER_EC_EXCHANGE_MELT_HISTORY_DB_ERROR_INSUFFICIENT_FUNDS,
-                                       NULL);
-  return TALER_MHD_REPLY_JSON_PACK (
-    connection,
-    MHD_HTTP_CONFLICT,
-    TALER_JSON_pack_ec (TALER_EC_EXCHANGE_MELT_INSUFFICIENT_FUNDS),
-    GNUNET_JSON_pack_data_auto ("coin_pub",
-                                coin_pub),
-    TALER_JSON_pack_amount ("original_value",
-                            coin_value),
-    TALER_JSON_pack_amount ("residual_value",
-                            residual),
-    TALER_JSON_pack_amount ("requested_value",
-                            requested),
-    GNUNET_JSON_pack_array_steal ("history",
-                                  history));
-}
-
-
 /**
  * Send a response to a "melt" request.
  *
@@ -165,124 +115,6 @@ struct MeltContext
 };
 
 
-/**
- * Check that the coin has sufficient funds left for the selected
- * melt operation.
- *
- * @param connection the connection to send errors to
- * @param[in,out] rmc melt context
- * @param[out] mhd_ret status code to return to MHD on hard error
- * @return transaction status code
- */
-static enum GNUNET_DB_QueryStatus
-refresh_check_melt (struct MHD_Connection *connection,
-                    struct MeltContext *rmc,
-                    MHD_RESULT *mhd_ret)
-{
-  struct TALER_EXCHANGEDB_TransactionList *tl;
-  struct TALER_Amount spent;
-  enum GNUNET_DB_QueryStatus qs;
-
-  /* Start with cost of this melt transaction */
-  spent = rmc->refresh_session.amount_with_fee;
-
-  /* get historic transaction costs of this coin, including recoups as
-     we might be a zombie coin */
-  qs = TEH_plugin->get_coin_transactions (TEH_plugin->cls,
-                                          &rmc->refresh_session.coin.coin_pub,
-                                          GNUNET_YES,
-                                          &tl);
-  if (0 > qs)
-  {
-    if (GNUNET_DB_STATUS_HARD_ERROR == qs)
-      *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                             MHD_HTTP_INTERNAL_SERVER_ERROR,
-                                             TALER_EC_GENERIC_DB_FETCH_FAILED,
-                                             "coin transaction history");
-    return qs;
-  }
-  if (rmc->zombie_required)
-  {
-    /* The denomination key is only usable for a melt if this is a true
-       zombie coin, i.e. it was refreshed and the resulting fresh coin was
-       then recouped. Check that this is truly the case. */
-    for (struct TALER_EXCHANGEDB_TransactionList *tp = tl;
-         NULL != tp;
-         tp = tp->next)
-    {
-      if (TALER_EXCHANGEDB_TT_OLD_COIN_RECOUP == tp->type)
-      {
-        rmc->zombie_required = false; /* clear flag: was satisfied! */
-        break;
-      }
-    }
-    if (rmc->zombie_required)
-    {
-      /* zombie status not satisfied */
-      GNUNET_break_op (0);
-      TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
-                                              tl);
-      *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                             MHD_HTTP_BAD_REQUEST,
-                                             TALER_EC_EXCHANGE_MELT_COIN_EXPIRED_NO_ZOMBIE,
-                                             NULL);
-      return GNUNET_DB_STATUS_HARD_ERROR;
-    }
-  }
-  if (GNUNET_OK !=
-      TALER_EXCHANGEDB_calculate_transaction_list_totals (tl,
-                                                          &spent,
-                                                          &spent))
-  {
-    GNUNET_break (0);
-    TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
-                                            tl);
-    *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                           MHD_HTTP_INTERNAL_SERVER_ERROR,
-                                           TALER_EC_EXCHANGE_MELT_COIN_HISTORY_COMPUTATION_FAILED,
-                                           NULL);
-    return GNUNET_DB_STATUS_HARD_ERROR;
-  }
-
-  /* Refuse to refresh when the coin's value is insufficient
-     for the cost of all transactions. */
-  if (0 > TALER_amount_cmp (&rmc->coin_value,
-                            &spent))
-  {
-    struct TALER_Amount coin_residual;
-    struct TALER_Amount spent_already;
-
-    /* First subtract the melt cost from 'spent' to
-       compute the total amount already spent of the coin */
-    GNUNET_assert (0 <=
-                   TALER_amount_subtract (&spent_already,
-                                          &spent,
-                                          &rmc->refresh_session.amount_with_fee));
-    /* The residual coin value is the original coin value minus
-       what we have spent (before the melt) */
-    GNUNET_assert (0 <=
-                   TALER_amount_subtract (&coin_residual,
-                                          &rmc->coin_value,
-                                          &spent_already));
-    *mhd_ret = reply_melt_insufficient_funds (
-      connection,
-      &rmc->refresh_session.coin.coin_pub,
-      &rmc->coin_value,
-      tl,
-      &rmc->refresh_session.amount_with_fee,
-      &coin_residual);
-    TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
-                                            tl);
-    return GNUNET_DB_STATUS_HARD_ERROR;
-  }
-
-  /* we're good, coin has sufficient funds to be melted */
-  TEH_plugin->free_coin_transaction_list (TEH_plugin->cls,
-                                          tl);
-  return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
-}
-
-
 /**
  * Execute a "melt".  We have been given a list of valid
  * coins and a request to melt them into the given @a
@@ -311,43 +143,11 @@ melt_transaction (void *cls,
   enum GNUNET_DB_QueryStatus qs;
   uint32_t noreveal_index;
 
-  /* Check if we already created a matching refresh_session */
-  qs = TEH_plugin->get_melt_index (TEH_plugin->cls,
-                                   &rmc->refresh_session.rc,
-                                   &noreveal_index);
-  if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
-  {
-    TALER_LOG_DEBUG ("Coin was previously melted, returning old reply\n");
-    *mhd_ret = reply_melt_success (connection,
-                                   &rmc->refresh_session.rc,
-                                   noreveal_index);
-    /* Note: we return "hard error" to ensure the wrapper
-       does not retry the transaction, and to also not generate
-       a "fresh" response (as we would on "success") */
-    return GNUNET_DB_STATUS_HARD_ERROR;
-  }
-  if (0 > qs)
-  {
-    if (GNUNET_DB_STATUS_HARD_ERROR == qs)
-      *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                             MHD_HTTP_INTERNAL_SERVER_ERROR,
-                                             TALER_EC_GENERIC_DB_FETCH_FAILED,
-                                             "melt index");
-    return qs;
-  }
-
-  /* check coin has enough funds remaining on it to cover melt cost */
-  qs = refresh_check_melt (connection,
-                           rmc,
-                           mhd_ret);
-  if (0 > qs)
-    return qs; /* if we failed, tell caller */
-
   /* pick challenge and persist it */
   rmc->refresh_session.noreveal_index
     = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_STRONG,
                                 TALER_CNC_KAPPA);
-  if (0 >=
+  if (0 >
       (qs = TEH_plugin->insert_melt (TEH_plugin->cls,
                                      &rmc->refresh_session)))
   {
@@ -361,7 +161,47 @@ melt_transaction (void *cls,
     }
     return qs;
   }
-  return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
+  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+  {
+    /* Check if we already created a matching refresh_session */
+    qs = TEH_plugin->get_melt_index (TEH_plugin->cls,
+                                     &rmc->refresh_session.rc,
+                                     &noreveal_index);
+    if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
+    {
+      TALER_LOG_DEBUG ("Coin was previously melted, returning old reply\n");
+      *mhd_ret = reply_melt_success (connection,
+                                     &rmc->refresh_session.rc,
+                                     noreveal_index);
+      /* Note: we return "hard error" to ensure the wrapper
+         does not retry the transaction, and to also not generate
+         a "fresh" response (as we would on "success") */
+      return GNUNET_DB_STATUS_HARD_ERROR;
+    }
+    if (0 > qs)
+    {
+      if (GNUNET_DB_STATUS_HARD_ERROR == qs)
+        *mhd_ret = TALER_MHD_reply_with_error (connection,
+                                               MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                               TALER_EC_GENERIC_DB_FETCH_FAILED,
+                                               "melt index");
+      return qs;
+    }
+    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+    {
+      /* Conflict on insert, but record does not exist?
+         That makes no sense. */
+      GNUNET_break (0);
+      return GNUNET_DB_STATUS_HARD_ERROR;
+    }
+  }
+  return TEH_check_coin_balance (connection,
+                                 &rmc->refresh_session.coin.coin_pub,
+                                 &rmc->coin_value,
+                                 &rmc->refresh_session.amount_with_fee,
+                                 true,
+                                 rmc->zombie_required,
+                                 mhd_ret);
 }
 
 
@@ -380,6 +220,16 @@ static MHD_RESULT
 handle_melt (struct MHD_Connection *connection,
              struct MeltContext *rmc)
 {
+  if (GNUNET_SYSERR ==
+      TEH_plugin->preflight (TEH_plugin->cls))
+  {
+    GNUNET_break (0);
+    return TALER_MHD_reply_with_error (connection,
+                                       MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                       TALER_EC_GENERIC_DB_START_FAILED,
+                                       "preflight failure");
+  }
+
   /* verify signature of coin for melt operation */
   {
     struct TALER_RefreshMeltCoinAffirmationPS body = {
@@ -432,6 +282,7 @@ handle_melt (struct MHD_Connection *connection,
     if (GNUNET_OK !=
         TEH_DB_run_transaction (connection,
                                 "run melt",
+                                TEH_MT_MELT,
                                 &mhd_ret,
                                 &melt_transaction,
                                 rmc))

src/exchange/taler-exchange-httpd_metrics.c

@@ -0,0 +1,106 @@
+/*
+  This file is part of TALER
+  Copyright (C) 2015-2021 Taler Systems SA
+
+  TALER is free software; you can redistribute it and/or modify it under the
+  terms of the GNU Affero General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License along with
+  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file taler-exchange-httpd_metrics.c
+ * @brief Handle /metrics requests
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include <gnunet/gnunet_json_lib.h>
+#include "taler_dbevents.h"
+#include "taler-exchange-httpd_responses.h"
+#include "taler-exchange-httpd_keys.h"
+#include "taler-exchange-httpd_metrics.h"
+#include "taler_json_lib.h"
+#include "taler_mhd_lib.h"
+#include <jansson.h>
+
+
+unsigned long long TEH_METRICS_num_requests[TEH_MT_COUNT];
+
+unsigned long long TEH_METRICS_num_conflict[TEH_MT_COUNT];
+
+
+MHD_RESULT
+TEH_handler_metrics (struct TEH_RequestContext *rc,
+                     const char *const args[])
+{
+  char *reply;
+  struct MHD_Response *resp;
+  MHD_RESULT ret;
+
+  (void) args;
+  GNUNET_asprintf (&reply,
+                   "# HELP taler_exchange_serialization_failures "
+                   " number of database serialization errors by type\n"
+                   "# TYPE taler_exchange_serialization_failures counter\n"
+                   "taler_exchange_serialization_failures{type=\"%s\"} %llu\n"
+                   "taler_exchange_serialization_failures{type=\"%s\"} %llu\n"
+                   "taler_exchange_serialization_failures{type=\"%s\"} %llu\n"
+                   "taler_exchange_serialization_failures{type=\"%s\"} %llu\n"
+                   "taler_exchange_serialization_failures{type=\"%s\"} %llu\n"
+                   "taler_exchange_serialization_failures{type=\"%s\"} %llu\n"
+                   "taler_exchange_serialization_failures{type=\"%s\"} %llu\n"
+                   "# HELP taler_exchange_received_requests "
+                   " number of received requests by type\n"
+                   "# TYPE taler_exchange_received_requests counter\n"
+                   "taler_exchange_received_requests{type=\"%s\"} %llu\n"
+                   "taler_exchange_received_requests{type=\"%s\"} %llu\n"
+                   "taler_exchange_received_requests{type=\"%s\"} %llu\n"
+                   "taler_exchange_received_requests{type=\"%s\"} %llu\n"
+                   "taler_exchange_received_requests{type=\"%s\"} %llu\n"
+                   "taler_exchange_received_requests{type=\"%s\"} %llu\n"
+                   "taler_exchange_received_requests{type=\"%s\"} %llu\n",
+                   "other",
+                   TEH_METRICS_num_conflict[TEH_MT_OTHER],
+                   "deposit",
+                   TEH_METRICS_num_conflict[TEH_MT_DEPOSIT],
+                   "withdraw",
+                   TEH_METRICS_num_conflict[TEH_MT_WITHDRAW],
+                   "melt",
+                   TEH_METRICS_num_conflict[TEH_MT_MELT],
+                   "reveal-precheck",
+                   TEH_METRICS_num_conflict[TEH_MT_REVEAL_PRECHECK],
+                   "reveal",
+                   TEH_METRICS_num_conflict[TEH_MT_REVEAL],
+                   "reveal-persist",
+                   TEH_METRICS_num_conflict[TEH_MT_REVEAL_PERSIST],
+                   "other",
+                   TEH_METRICS_num_requests[TEH_MT_OTHER],
+                   "deposit",
+                   TEH_METRICS_num_requests[TEH_MT_DEPOSIT],
+                   "withdraw",
+                   TEH_METRICS_num_requests[TEH_MT_WITHDRAW],
+                   "melt",
+                   TEH_METRICS_num_requests[TEH_MT_MELT],
+                   "reveal-precheck",
+                   TEH_METRICS_num_requests[TEH_MT_REVEAL_PRECHECK],
+                   "reveal",
+                   TEH_METRICS_num_requests[TEH_MT_REVEAL],
+                   "reveal-persist",
+                   TEH_METRICS_num_requests[TEH_MT_REVEAL_PERSIST]);
+  resp = MHD_create_response_from_buffer (strlen (reply),
+                                          reply,
+                                          MHD_RESPMEM_MUST_FREE);
+  ret = MHD_queue_response (rc->connection,
+                            MHD_HTTP_OK,
+                            resp);
+  MHD_destroy_response (resp);
+  return ret;
+}
+
+
+/* end of taler-exchange-httpd_metrics.c */

src/exchange/taler-exchange-httpd_metrics.h

@@ -0,0 +1,69 @@
+/*
+  This file is part of TALER
+  Copyright (C) 2014--2021 Taler Systems SA
+
+  TALER is free software; you can redistribute it and/or modify it under the
+  terms of the GNU Affero General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License along with
+  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file taler-exchange-httpd_metrics.h
+ * @brief Handle /metrics requests
+ * @author Christian Grothoff
+ */
+#ifndef TALER_EXCHANGE_HTTPD_METRICS_H
+#define TALER_EXCHANGE_HTTPD_METRICS_H
+
+#include <gnunet/gnunet_util_lib.h>
+#include <microhttpd.h>
+#include "taler-exchange-httpd.h"
+
+
+/**
+ * Request types for which we collect metrics.
+ */
+enum TEH_MetricType
+{
+  TEH_MT_OTHER = 0,
+  TEH_MT_DEPOSIT = 1,
+  TEH_MT_WITHDRAW = 2,
+  TEH_MT_MELT = 3,
+  TEH_MT_REVEAL_PRECHECK = 4,
+  TEH_MT_REVEAL = 5,
+  TEH_MT_REVEAL_PERSIST = 6,
+  TEH_MT_COUNT = 7 /* MUST BE LAST! */
+};
+
+
+/**
+ * Number of requests handled of the respective type.
+ */
+extern unsigned long long TEH_METRICS_num_requests[TEH_MT_COUNT];
+
+/**
+ * Number of serialization errors encountered when
+ * handling requests of the respective type.
+ */
+extern unsigned long long TEH_METRICS_num_conflict[TEH_MT_COUNT];
+
+
+/**
+ * Handle a "/metrics" request.
+ *
+ * @param rc request context
+ * @param args array of additional options (must be empty for this function)
+ * @return MHD result code
+ */
+MHD_RESULT
+TEH_handler_metrics (struct TEH_RequestContext *rc,
+                     const char *const args[]);
+
+
+#endif

src/exchange/taler-exchange-httpd_recoup.c

@@ -1,6 +1,6 @@
 /*
   This file is part of TALER
-  Copyright (C) 2017-2020 Taler Systems SA
+  Copyright (C) 2017-2021 Taler Systems SA
 
   TALER is free software; you can redistribute it and/or modify it under the
   terms of the GNU Affero General Public License as published by the Free Software
@@ -28,6 +28,7 @@
 #include <pthread.h>
 #include "taler_json_lib.h"
 #include "taler_mhd_lib.h"
+#include "taler-exchange-httpd_db.h"
 #include "taler-exchange-httpd_recoup.h"
 #include "taler-exchange-httpd_responses.h"
 #include "taler-exchange-httpd_keys.h"
@@ -94,13 +95,14 @@ struct RecoupContext
   struct GNUNET_TIME_Absolute now;
 
   /**
-   * #GNUNET_YES if the client claims the coin originated from a refresh.
+   * true if the client claims the coin originated from a refresh.
    */
-  int refreshed;
+  bool refreshed;
 
 };
 
 
+// FIXME: this code should be simplified by using TEH_check_coin_balance()
 /**
  * Execute a "recoup".  The validity of the coin and signature have
  * already been checked.  The database must now check that the coin is
@@ -128,64 +130,10 @@ recoup_transaction (void *cls,
   struct TALER_Amount spent;
   struct TALER_Amount recouped;
   enum GNUNET_DB_QueryStatus qs;
-  int existing_recoup_found;
+  bool existing_recoup_found;
-
-  /* make sure coin is 'known' in database */
-  qs = TEH_make_coin_known (pc->coin,
-                            connection,
-                            mhd_ret);
-  if (qs < 0)
-    return qs;
 
   /* Check whether a recoup is allowed, and if so, to which
      reserve / account the money should go */
-  if (pc->refreshed)
-  {
-    qs = TEH_plugin->get_old_coin_by_h_blind (TEH_plugin->cls,
-                                              &pc->h_blind,
-                                              &pc->target.old_coin_pub);
-    if (0 > qs)
-    {
-      if (GNUNET_DB_STATUS_HARD_ERROR == qs)
-      {
-        GNUNET_break (0);
-        *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                               MHD_HTTP_INTERNAL_SERVER_ERROR,
-                                               TALER_EC_GENERIC_DB_FETCH_FAILED,
-                                               "old coin by h_blind");
-      }
-      return qs;
-    }
-  }
-  else
-  {
-    qs = TEH_plugin->get_reserve_by_h_blind (TEH_plugin->cls,
-                                             &pc->h_blind,
-                                             &pc->target.reserve_pub);
-    if (0 > qs)
-    {
-      if (GNUNET_DB_STATUS_HARD_ERROR == qs)
-      {
-        GNUNET_break (0);
-        *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                               MHD_HTTP_INTERNAL_SERVER_ERROR,
-                                               TALER_EC_GENERIC_DB_FETCH_FAILED,
-                                               "reserve by h_blind");
-      }
-      return qs;
-    }
-  }
-  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                "Recoup requested for unknown envelope %s\n",
-                GNUNET_h2s (&pc->h_blind.hash));
-    *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                           MHD_HTTP_NOT_FOUND,
-                                           TALER_EC_EXCHANGE_RECOUP_WITHDRAW_NOT_FOUND,
-                                           NULL);
-    return GNUNET_DB_STATUS_HARD_ERROR;
-  }
 
   /* Calculate remaining balance, including recoups already applied. */
   qs = TEH_plugin->get_coin_transactions (TEH_plugin->cls,
@@ -212,7 +160,7 @@ recoup_transaction (void *cls,
                  TALER_amount_set_zero (pc->value.currency,
                                         &recouped));
   /* Check if this coin has been recouped already at least once */
-  existing_recoup_found = GNUNET_NO;
+  existing_recoup_found = false;
   for (struct TALER_EXCHANGEDB_TransactionList *pos = tl;
        NULL != pos;
        pos = pos->next)
@@ -220,7 +168,7 @@ recoup_transaction (void *cls,
     if ( (TALER_EXCHANGEDB_TT_RECOUP == pos->type) ||
          (TALER_EXCHANGEDB_TT_RECOUP_REFRESH == pos->type) )
     {
-      existing_recoup_found = GNUNET_YES;
+      existing_recoup_found = true;
       break;
     }
   }
@@ -258,8 +206,7 @@ recoup_transaction (void *cls,
                                            NULL);
     return GNUNET_DB_STATUS_HARD_ERROR;
   }
-  if ( (0 == pc->amount.fraction) &&
+  if (TALER_amount_is_zero (&pc->amount))
-       (0 == pc->amount.value) )
   {
     /* Recoup has no effect: coin fully spent! */
     enum GNUNET_DB_QueryStatus ret;
@@ -338,7 +285,7 @@ recoup_transaction (void *cls,
  * @param coin information about the coin
  * @param coin_bks blinding data of the coin (to be checked)
  * @param coin_sig signature of the coin
- * @param refreshed #GNUNET_YES if the coin was refreshed
+ * @param refreshed true if the coin was refreshed
  * @return MHD result code
  */
 static MHD_RESULT
@@ -347,7 +294,7 @@ verify_and_execute_recoup (
   const struct TALER_CoinPublicInfo *coin,
   const union TALER_DenominationBlindingKeyP *coin_bks,
   const struct TALER_CoinSpendSignatureP *coin_sig,
-  int refreshed)
+  bool refreshed)
 {
   struct RecoupContext pc;
   const struct TEH_DenominationKey *dk;
@@ -466,17 +413,79 @@ verify_and_execute_recoup (
     GNUNET_free (coin_ev);
   }
 
-  /* Perform actual recoup transaction */
   pc.coin_sig = coin_sig;
   pc.coin_bks = coin_bks;
   pc.coin = coin;
   pc.refreshed = refreshed;
+
+  {
+    MHD_RESULT mhd_ret = MHD_NO;
+    enum GNUNET_DB_QueryStatus qs;
+
+    /* make sure coin is 'known' in database */
+    qs = TEH_make_coin_known (coin,
+                              connection,
+                              &mhd_ret);
+    /* no transaction => no serialization failures should be possible */
+    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs);
+    if (qs < 0)
+      return mhd_ret;
+  }
+
+  {
+    enum GNUNET_DB_QueryStatus qs;
+
+    if (pc.refreshed)
+    {
+      qs = TEH_plugin->get_old_coin_by_h_blind (TEH_plugin->cls,
+                                                &pc.h_blind,
+                                                &pc.target.old_coin_pub);
+      if (0 > qs)
+      {
+        GNUNET_break (0);
+        return TALER_MHD_reply_with_error (
+          connection,
+          MHD_HTTP_INTERNAL_SERVER_ERROR,
+          TALER_EC_GENERIC_DB_FETCH_FAILED,
+          "old coin by h_blind");
+      }
+    }
+    else
+    {
+      qs = TEH_plugin->get_reserve_by_h_blind (TEH_plugin->cls,
+                                               &pc.h_blind,
+                                               &pc.target.reserve_pub);
+      if (0 > qs)
+      {
+        GNUNET_break (0);
+        return TALER_MHD_reply_with_error (
+          connection,
+          MHD_HTTP_INTERNAL_SERVER_ERROR,
+          TALER_EC_GENERIC_DB_FETCH_FAILED,
+          "reserve by h_blind");
+      }
+    }
+    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "Recoup requested for unknown envelope %s\n",
+                  GNUNET_h2s (&pc.h_blind.hash));
+      return TALER_MHD_reply_with_error (
+        connection,
+        MHD_HTTP_NOT_FOUND,
+        TALER_EC_EXCHANGE_RECOUP_WITHDRAW_NOT_FOUND,
+        NULL);
+    }
+  }
+
+  /* Perform actual recoup transaction */
   {
     MHD_RESULT mhd_ret;
 
     if (GNUNET_OK !=
         TEH_DB_run_transaction (connection,
                                 "run recoup",
+                                TEH_MT_OTHER,
                                 &mhd_ret,
                                 &recoup_transaction,
                                 &pc))
@@ -521,7 +530,7 @@ TEH_handler_recoup (struct MHD_Connection *connection,
   struct TALER_CoinPublicInfo coin;
   union TALER_DenominationBlindingKeyP coin_bks;
   struct TALER_CoinSpendSignatureP coin_sig;
-  int refreshed = GNUNET_NO;
+  bool refreshed = false;
   struct GNUNET_JSON_Specification spec[] = {
     GNUNET_JSON_spec_fixed_auto ("denom_pub_hash",
                                  &coin.denom_pub_hash),
@@ -531,9 +540,9 @@ TEH_handler_recoup (struct MHD_Connection *connection,
                                  &coin_bks),
     GNUNET_JSON_spec_fixed_auto ("coin_sig",
                                  &coin_sig),
-    GNUNET_JSON_spec_mark_optional
+    GNUNET_JSON_spec_mark_optional (
-      (GNUNET_JSON_spec_boolean ("refreshed",
+      GNUNET_JSON_spec_bool ("refreshed",
-                                 &refreshed)),
+                             &refreshed)),
     GNUNET_JSON_spec_end ()
   };
 

src/exchange/taler-exchange-httpd_refreshes_reveal.c

@@ -105,6 +105,11 @@ struct RevealContext
    */
   struct TALER_TransferPrivateKeyP transfer_privs[TALER_CNC_KAPPA - 1];
 
+  /**
+   * Melt data for our session we got from the database for @e rc.
+   */
+  struct TALER_EXCHANGEDB_Melt melt;
+
   /**
    * Denominations being requested.
    */
@@ -266,35 +271,6 @@ refreshes_reveal_transaction (void *cls,
                               MHD_RESULT *mhd_ret)
 {
   struct RevealContext *rctx = cls;
-  struct TALER_EXCHANGEDB_Melt melt;
-  enum GNUNET_DB_QueryStatus qs;
-
-  /* Obtain basic information about the refresh operation and what
-     gamma we committed to. */
-  // FIXME: why do we do 'get_melt' twice?
-  qs = TEH_plugin->get_melt (TEH_plugin->cls,
-                             &rctx->rc,
-                             &melt);
-  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
-  {
-    *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                           MHD_HTTP_NOT_FOUND,
-                                           TALER_EC_EXCHANGE_REFRESHES_REVEAL_SESSION_UNKNOWN,
-                                           NULL);
-    return GNUNET_DB_STATUS_HARD_ERROR;
-  }
-  if (GNUNET_DB_STATUS_SOFT_ERROR == qs)
-    return qs;
-  if ( (GNUNET_DB_STATUS_HARD_ERROR == qs) ||
-       (melt.session.noreveal_index >= TALER_CNC_KAPPA) )
-  {
-    GNUNET_break (0);
-    *mhd_ret = TALER_MHD_reply_with_error (connection,
-                                           MHD_HTTP_INTERNAL_SERVER_ERROR,
-                                           TALER_EC_GENERIC_DB_FETCH_FAILED,
-                                           "melt");
-    return GNUNET_DB_STATUS_HARD_ERROR;
-  }
 
   /* Verify commitment */
   {
@@ -310,7 +286,7 @@ refreshes_reveal_transaction (void *cls,
     {
       struct TALER_RefreshCommitmentEntry *rce = &rcs[i];
 
-      if (i == melt.session.noreveal_index)
+      if (i == rctx->melt.session.noreveal_index)
       {
         /* Take these coin envelopes from the client */
         rce->transfer_pub = rctx->gamma_tp;
@@ -327,7 +303,7 @@ refreshes_reveal_transaction (void *cls,
         GNUNET_CRYPTO_ecdhe_key_get_public (&tpriv->ecdhe_priv,
                                             &rce->transfer_pub.ecdhe_pub);
         TALER_link_reveal_transfer_secret (tpriv,
-                                           &melt.session.coin.coin_pub,
+                                           &rctx->melt.session.coin.coin_pub,
                                            &ts);
         rce->new_coins = GNUNET_new_array (rctx->num_fresh_coins,
                                            struct TALER_RefreshCoinData);
@@ -356,15 +332,15 @@ refreshes_reveal_transaction (void *cls,
                                   TALER_CNC_KAPPA,
                                   rctx->num_fresh_coins,
                                   rcs,
-                                  &melt.session.coin.coin_pub,
+                                  &rctx->melt.session.coin.coin_pub,
-                                  &melt.session.amount_with_fee);
+                                  &rctx->melt.session.amount_with_fee);
 
     /* Free resources allocated above */
     for (unsigned int i = 0; i<TALER_CNC_KAPPA; i++)
     {
       struct TALER_RefreshCommitmentEntry *rce = &rcs[i];
 
-      if (i == melt.session.noreveal_index)
+      if (i == rctx->melt.session.noreveal_index)
         continue; /* This offset is special: not allocated! */
       for (unsigned int j = 0; j<rctx->num_fresh_coins; j++)
       {
@@ -395,7 +371,7 @@ refreshes_reveal_transaction (void *cls,
   {
     struct TALER_Amount refresh_cost;
 
-    refresh_cost = melt.melt_fee;
+    refresh_cost = rctx->melt.melt_fee;
     for (unsigned int i = 0; i<rctx->num_fresh_coins; i++)
     {
       struct TALER_Amount total;
@@ -418,7 +394,7 @@ refreshes_reveal_transaction (void *cls,
       }
     }
     if (0 < TALER_amount_cmp (&refresh_cost,
-                              &melt.session.amount_with_fee))
+                              &rctx->melt.session.amount_with_fee))
     {
       GNUNET_break_op (0);
       *mhd_ret = TALER_MHD_reply_with_error (connection,
@@ -505,7 +481,6 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection,
   struct TALER_DenominationHash dk_h[num_fresh_coins];
   struct TALER_RefreshCoinData rcds[num_fresh_coins];
   struct TALER_CoinSpendSignatureP link_sigs[num_fresh_coins];
-  struct TALER_EXCHANGEDB_Melt melt;
   enum GNUNET_GenericReturnValue res;
   MHD_RESULT ret;
   struct TEH_KeyStateHandle *ksh;
@@ -612,11 +587,10 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection,
   {
     enum GNUNET_DB_QueryStatus qs;
 
-    // FIXME: why do we do 'get_melt' twice?
     if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
         (qs = TEH_plugin->get_melt (TEH_plugin->cls,
                                     &rctx->rc,
-                                    &melt)))
+                                    &rctx->melt)))
     {
       switch (qs)
       {
@@ -643,6 +617,17 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection,
       }
       goto cleanup;
     }
+    /* Obtain basic information about the refresh operation and what
+       gamma we committed to. */
+    if (rctx->melt.session.noreveal_index >= TALER_CNC_KAPPA)
+    {
+      GNUNET_break (0);
+      ret = TALER_MHD_reply_with_error (connection,
+                                        MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                        TALER_EC_GENERIC_DB_FETCH_FAILED,
+                                        "melt");
+      goto cleanup;
+    }
   }
   /* Parse link signatures array */
   for (unsigned int i = 0; i<num_fresh_coins; i++)
@@ -666,7 +651,7 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection,
           &rctx->gamma_tp,
           rcds[i].coin_ev,
           rcds[i].coin_ev_size,
-          &melt.session.coin.coin_pub,
+          &rctx->melt.session.coin.coin_pub,
           &link_sigs[i]))
     {
       GNUNET_break_op (0);
@@ -724,6 +709,7 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection,
     if ( (GNUNET_OK ==
           TEH_DB_run_transaction (connection,
                                   "reveal pre-check",
+                                  TEH_MT_REVEAL_PRECHECK,
                                   &ret,
                                   &refreshes_reveal_preflight,
                                   rctx)) &&
@@ -745,6 +731,7 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection,
     if (GNUNET_OK !=
         TEH_DB_run_transaction (connection,
                                 "run reveal",
+                                TEH_MT_REVEAL,
                                 &ret,
                                 &refreshes_reveal_transaction,
                                 rctx))
@@ -756,6 +743,7 @@ resolve_refreshes_reveal_denominations (struct MHD_Connection *connection,
     if (GNUNET_OK ==
         TEH_DB_run_transaction (connection,
                                 "persist reveal",
+                                TEH_MT_REVEAL_PERSIST,
                                 &ret,
                                 &refreshes_reveal_persist,
                                 rctx))

src/exchange/taler-exchange-httpd_refund.c

@@ -447,6 +447,7 @@ verify_and_execute_refund (struct MHD_Connection *connection,
     if (GNUNET_OK !=
         TEH_DB_run_transaction (connection,
                                 "run refund",
+                                TEH_MT_OTHER,
                                 &mhd_ret,
                                 &refund_transaction,
                                 (void *) refund))

src/exchange/taler-exchange-httpd_reserves_get.c

@@ -315,6 +315,7 @@ TEH_handler_reserves_get (struct TEH_RequestContext *rc,
   if (GNUNET_OK !=
       TEH_DB_run_transaction (rc->connection,
                               "get reserve history",
+                              TEH_MT_OTHER,
                               &mhd_ret,
                               &reserve_history_transaction,
                               &rsc))

src/exchange/taler-exchange-httpd_transfers_get.c

@@ -515,6 +515,7 @@ TEH_handler_transfers_get (struct TEH_RequestContext *rc,
   if (GNUNET_OK !=
       TEH_DB_run_transaction (rc->connection,
                               "run transfers GET",
+                              TEH_MT_OTHER,
                               &mhd_ret,
                               &get_transfer_deposits,
                               &ctx))

src/exchange/taler-exchange-httpd_withdraw.c

@@ -521,6 +521,7 @@ TEH_handler_withdraw (struct TEH_RequestContext *rc,
     if (GNUNET_OK !=
         TEH_DB_run_transaction (rc->connection,
                                 "run withdraw",
+                                TEH_MT_WITHDRAW,
                                 &mhd_ret,
                                 &withdraw_transaction,
                                 &wc))

src/exchange/taler-exchange-wirewatch.c

@@ -125,6 +125,11 @@ struct WireAccount
    */
   bool delay;
 
+  /**
+   * Did we start a transaction yet?
+   */
+  bool started_transaction;
+
 };
 
 
@@ -229,6 +234,12 @@ shutdown_task (void *cls)
       GNUNET_CONTAINER_DLL_remove (wa_head,
                                    wa_tail,
                                    wa);
+      if (wa->started_transaction)
+      {
+        db_plugin->rollback (db_plugin->cls);
+        wa->started_transaction = false;
+      }
+      // FIXME: delete shard lock here (#7124)
       GNUNET_free (wa->job_name);
       GNUNET_free (wa);
     }
@@ -293,7 +304,7 @@ add_account_cb (void *cls,
  *
  * @return #GNUNET_OK on success
  */
-static int
+static enum GNUNET_GenericReturnValue
 exchange_serve_process_config (void)
 {
   if (GNUNET_OK !=
@@ -351,6 +362,7 @@ static void
 handle_soft_error (struct WireAccount *wa)
 {
   db_plugin->rollback (db_plugin->cls);
+  wa->started_transaction = false;
   if (1 < wa->batch_size)
   {
     wa->batch_thresh = wa->batch_size;
@@ -365,6 +377,94 @@ handle_soft_error (struct WireAccount *wa)
 }
 
 
+/**
+ * We are done with a shard, move on to the next one.
+ *
+ * @param wa wire account for which we completed a shard
+ */
+static void
+shard_completed (struct WireAccount *wa)
+{
+  /* transaction success, update #last_row_off */
+  wa->batch_start = wa->latest_row_off;
+  if (wa->batch_size < MAXIMUM_BATCH_SIZE)
+  {
+    int delta;
+
+    delta = ((int) wa->batch_thresh - (int) wa->batch_size) / 4;
+    if (delta < 0)
+      delta = -delta;
+    wa->batch_size = GNUNET_MIN (MAXIMUM_BATCH_SIZE,
+                                 wa->batch_size + delta + 1);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Increasing batch size to %llu\n",
+                (unsigned long long) wa->batch_size);
+  }
+  if (wa->delay)
+  {
+    wa->delayed_until
+      = GNUNET_TIME_relative_to_absolute (wirewatch_idle_sleep_interval);
+    wa_pos = wa_pos->next;
+    if (NULL == wa_pos)
+      wa_pos = wa_head;
+    GNUNET_assert (NULL != wa_pos);
+  }
+  GNUNET_assert (NULL == task);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Will look for more transfers in %s\n",
+              GNUNET_STRINGS_relative_time_to_string (
+                GNUNET_TIME_absolute_get_remaining (wa_pos->delayed_until),
+                GNUNET_YES));
+  task = GNUNET_SCHEDULER_add_at (wa_pos->delayed_until,
+                                  &find_transfers,
+                                  NULL);
+}
+
+
+/**
+ * We are finished with the current shard. Update the database, marking the
+ * shard as finished.
+ *
+ * @param wa wire account to commit for
+ * @return true on success
+ */
+static bool
+mark_shard_done (struct WireAccount *wa)
+{
+  enum GNUNET_DB_QueryStatus qs;
+
+  if (wa->shard_end > wa->latest_row_off)
+    return false; /* actually, not done! */
+  /* shard is complete, mark this as well */
+  qs = db_plugin->complete_shard (db_plugin->cls,
+                                  wa->job_name,
+                                  wa->shard_start,
+                                  wa->shard_end);
+  switch (qs)
+  {
+  case GNUNET_DB_STATUS_HARD_ERROR:
+    GNUNET_break (0);
+    db_plugin->rollback (db_plugin->cls);
+    GNUNET_SCHEDULER_shutdown ();
+    return false;
+  case GNUNET_DB_STATUS_SOFT_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Got DB soft error for complete_shard. Rolling back.\n");
+    handle_soft_error (wa);
+    return false;
+  case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
+    /* already existed, ok, let's just continue */
+    break;
+  case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
+    /* normal case */
+    shard_delay = GNUNET_TIME_absolute_get_duration (wa->shard_start_time);
+
+    break;
+  }
+  return true;
+}
+
+
 /**
  * We are finished with the current transaction, try
  * to commit and then schedule the next iteration.
@@ -376,35 +476,8 @@ do_commit (struct WireAccount *wa)
 {
   enum GNUNET_DB_QueryStatus qs;
 
-  if (wa->shard_end <= wa->latest_row_off)
+  wa->started_transaction = false;
-  {
+  mark_shard_done (wa);
-    /* shard is complete, mark this as well */
-    qs = db_plugin->complete_shard (db_plugin->cls,
-                                    wa->job_name,
-                                    wa->shard_start,
-                                    wa->shard_end);
-    switch (qs)
-    {
-    case GNUNET_DB_STATUS_HARD_ERROR:
-      GNUNET_break (0);
-      db_plugin->rollback (db_plugin->cls);
-      GNUNET_SCHEDULER_shutdown ();
-      return;
-    case GNUNET_DB_STATUS_SOFT_ERROR:
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  "Got DB soft error for complete_shard. Rolling back.\n");
-      handle_soft_error (wa);
-      return;
-    case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
-      /* already existed, ok, let's just continue */
-      break;
-    case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
-      /* normal case */
-      shard_delay = GNUNET_TIME_absolute_get_duration (wa->shard_start_time);
-
-      break;
-    }
-  }
   qs = db_plugin->commit (db_plugin->cls);
   switch (qs)
   {
@@ -421,43 +494,7 @@ do_commit (struct WireAccount *wa)
     /* normal case */
     break;
   }
-  /* transaction success, update #last_row_off */
+  shard_completed (wa);
-  wa->batch_start = wa->latest_row_off;
-  if (wa->batch_size < MAXIMUM_BATCH_SIZE)
-  {
-    int delta;
-
-    delta = ((int) wa->batch_thresh - (int) wa->batch_size) / 4;
-    if (delta < 0)
-      delta = -delta;
-    wa->batch_size = GNUNET_MIN (MAXIMUM_BATCH_SIZE,
-                                 wa->batch_size + delta + 1);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Increasing batch size to %llu\n",
-                (unsigned long long) wa->batch_size);
-  }
-  if ( (wa->delay) &&
-       (test_mode) &&
-       (NULL == wa->next) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                "Shutdown due to test mode!\n");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  if (wa->delay)
-  {
-    wa->delayed_until
-      = GNUNET_TIME_relative_to_absolute (wirewatch_idle_sleep_interval);
-    wa_pos = wa_pos->next;
-    if (NULL == wa_pos)
-      wa_pos = wa_head;
-    GNUNET_assert (NULL != wa_pos);
-  }
-  GNUNET_assert (NULL == task);
-  task = GNUNET_SCHEDULER_add_at (wa_pos->delayed_until,
-                                  &find_transfers,
-                                  NULL);
 }
 
 
@@ -473,7 +510,7 @@ do_commit (struct WireAccount *wa)
  * @param json raw JSON response
  * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
  */
-static int
+static enum GNUNET_GenericReturnValue
 history_cb (void *cls,
             unsigned int http_status,
             enum TALER_ErrorCode ec,
@@ -495,9 +532,33 @@ history_cb (void *cls,
                   (unsigned int) ec,
                   http_status);
     }
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+    else
-                "End of list. Committing progress!\n");
+    {
-    do_commit (wa);
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "History response complete\n");
+    }
+    if (wa->started_transaction)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "End of list. Committing progress!\n");
+      do_commit (wa);
+    }
+    else
+    {
+      if ( (wa->delay) &&
+           (test_mode) &&
+           (NULL == wa->next) )
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "Shutdown due to test mode!\n");
+        GNUNET_SCHEDULER_shutdown ();
+        return GNUNET_OK;
+      }
+      else
+      {
+        shard_completed (wa);
+      }
+    }
     return GNUNET_OK; /* will be ignored anyway */
   }
   if (serial_id < wa->latest_row_off)
@@ -507,7 +568,11 @@ history_cb (void *cls,
                 "Serial ID %llu not monotonic (got %llu before). Failing!\n",
                 (unsigned long long) serial_id,
                 (unsigned long long) wa->latest_row_off);
-    db_plugin->rollback (db_plugin->cls);
+    if (wa->started_transaction)
+    {
+      wa->started_transaction = false;
+      db_plugin->rollback (db_plugin->cls);
+    }
     GNUNET_SCHEDULER_shutdown ();
     wa->hh = NULL;
     return GNUNET_SYSERR;
@@ -521,10 +586,34 @@ history_cb (void *cls,
                 (unsigned long long) wa->shard_end);
     wa->latest_row_off = serial_id - 1;
     wa->delay = false;
-    do_commit (wa);
+    if (wa->started_transaction)
+    {
+      do_commit (wa);
+    }
+    else
+    {
+      if (mark_shard_done (wa))
+        shard_completed (wa);
+    }
     wa->hh = NULL;
     return GNUNET_SYSERR;
   }
+  if (! wa->started_transaction)
+  {
+    if (GNUNET_OK !=
+        db_plugin->start_read_committed (db_plugin->cls,
+                                         "wirewatch check for incoming wire transfers"))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Failed to start database transaction!\n");
+      global_ret = EXIT_FAILURE;
+      GNUNET_SCHEDULER_shutdown ();
+      wa->hh = NULL;
+      return GNUNET_SYSERR;
+    }
+    wa_pos->shard_start_time = GNUNET_TIME_absolute_get ();
+    wa->started_transaction = true;
+  }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Adding wire transfer over %s with (hashed) subject `%s'\n",
               TALER_amount2s (&details->amount),
@@ -546,6 +635,7 @@ history_cb (void *cls,
   case GNUNET_DB_STATUS_HARD_ERROR:
     GNUNET_break (0);
     db_plugin->rollback (db_plugin->cls);
+    wa->started_transaction = false;
     GNUNET_SCHEDULER_shutdown ();
     wa->hh = NULL;
     return GNUNET_SYSERR;
@@ -624,37 +714,39 @@ find_transfers (void *cls)
       return;
     case GNUNET_DB_STATUS_SOFT_ERROR:
       /* try again */
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Serialization error tying to obtain shard, will try again in %s!\n",
+                  GNUNET_STRINGS_relative_time_to_string (
+                    wirewatch_idle_sleep_interval,
+                    GNUNET_YES));
       task = GNUNET_SCHEDULER_add_delayed (wirewatch_idle_sleep_interval,
                                            &find_transfers,
                                            NULL);
       return;
     case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
       GNUNET_break (0);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "No shard available, will try again in %s!\n",
+                  GNUNET_STRINGS_relative_time_to_string (
+                    wirewatch_idle_sleep_interval,
+                    GNUNET_YES));
       task = GNUNET_SCHEDULER_add_delayed (wirewatch_idle_sleep_interval,
                                            &find_transfers,
                                            NULL);
       return;
     case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
-      wa_pos->shard_start_time = GNUNET_TIME_absolute_get ();
       wa_pos->shard_start = start;
       wa_pos->shard_end = end;
       wa_pos->batch_start = start;
       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  "Starting with shard at %llu\n",
+                  "Starting with shard at [%llu,%llu) locked for %s\n",
-                  (unsigned long long) start);
+                  (unsigned long long) start,
+                  (unsigned long long) end,
+                  GNUNET_STRINGS_relative_time_to_string (delay,
+                                                          GNUNET_YES));
       break;
     }
   }
-  if (GNUNET_OK !=
-      db_plugin->start_read_committed (db_plugin->cls,
-                                       "wirewatch check for incoming wire transfers"))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Failed to start database transaction!\n");
-    global_ret = EXIT_FAILURE;
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
 
   limit = GNUNET_MIN (wa_pos->batch_size,
                       wa_pos->shard_end - wa_pos->batch_start);
@@ -673,7 +765,11 @@ find_transfers (void *cls)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Failed to start request for account history!\n");
-    db_plugin->rollback (db_plugin->cls);
+    if (wa_pos->started_transaction)
+    {
+      db_plugin->rollback (db_plugin->cls);
+      wa_pos->started_transaction = false;
+    }
     global_ret = EXIT_FAILURE;
     GNUNET_SCHEDULER_shutdown ();
     return;

src/exchangedb/exchange-0001.sql

@@ -899,6 +899,194 @@ COMMENT ON FUNCTION exchange_do_withdraw_limit_check(INT8, INT8, INT8, INT4)
 
 
 
+CREATE OR REPLACE FUNCTION exchange_do_check_coin_balance(
+  IN denom_val INT8, -- value of the denomination of the coin
+  IN denom_frac INT4, -- value of the denomination of the coin
+  IN in_coin_pub BYTEA, -- coin public key
+  IN check_recoup BOOLEAN, -- do we need to check the recoup table?
+  IN zombie_required BOOLEAN, -- do we need a zombie coin?
+  OUT balance_ok BOOLEAN, -- balance satisfied?
+  OUT zombie_ok BOOLEAN) -- zombie satisfied?
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  coin_uuid INT8; -- known_coin_id of coin_pub
+DECLARE
+  tmp_val INT8; -- temporary result
+DECLARE
+  tmp_frac INT8; -- temporary result
+DECLARE
+  spent_val INT8; -- how much of coin was spent?
+DECLARE
+  spent_frac INT8; -- how much of coin was spent?
+DECLARE
+  unspent_val INT8; -- how much of coin was refunded?
+DECLARE
+  unspent_frac INT8; -- how much of coin was refunded?
+BEGIN
+
+-- Note: possible future optimization: get the coin_uuid from the previous
+-- 'ensure_coin_known' and pass that here instead of the coin_pub. Might help
+-- a tiny bit with performance.
+SELECT known_coin_id INTO coin_uuid
+  FROM known_coins
+ WHERE coin_pub=in_coin_pub;
+
+IF NOT FOUND
+THEN
+  -- coin unknown, should be impossible!
+  balance_ok=FALSE;
+  zombie_ok=FALSE;
+  ASSERT false, 'coin unknown';
+  RETURN;
+END IF;
+
+
+spent_val = 0;
+spent_frac = 0;
+unspent_val = denom_val;
+unspent_frac = denom_frac;
+
+SELECT
+   SUM(amount_with_fee_val) -- overflow here is not plausible
+  ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits
+  INTO
+   tmp_val
+  ,tmp_frac
+  FROM deposits
+ WHERE known_coin_id=coin_uuid;
+
+IF tmp_val IS NOT NULL
+THEN
+  spent_val = spent_val + tmp_val;
+  spent_frac = spent_frac + tmp_frac;
+END IF;
+
+SELECT
+   SUM(amount_with_fee_val) -- overflow here is not plausible
+  ,SUM(CAST(amount_with_fee_frac AS INT8)) -- compute using 64 bits
+  INTO
+   tmp_val
+  ,tmp_frac
+  FROM refresh_commitments
+ WHERE old_known_coin_id=coin_uuid;
+
+IF tmp_val IS NOT NULL
+THEN
+  spent_val = spent_val + tmp_val;
+  spent_frac = spent_frac + tmp_frac;
+END IF;
+
+
+SELECT
+   SUM(rf.amount_with_fee_val) -- overflow here is not plausible
+  ,SUM(CAST(rf.amount_with_fee_frac AS INT8)) -- compute using 64 bits
+  INTO
+   tmp_val
+  ,tmp_frac
+  FROM deposits
+  JOIN refunds rf
+    USING (deposit_serial_id)
+  WHERE
+    known_coin_id=coin_uuid;
+IF tmp_val IS NOT NULL
+THEN
+  unspent_val = unspent_val + tmp_val;
+  unspent_frac = unspent_frac + tmp_frac;
+END IF;
+
+-- Note: even if 'check_recoup' is true, the tables below
+-- are in practice likely empty (as they only apply if
+-- the exchange (ever) had to revoke keys).
+IF check_recoup
+THEN
+
+  SELECT
+     SUM(amount_val) -- overflow here is not plausible
+    ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits
+    INTO
+     tmp_val
+    ,tmp_frac
+    FROM recoup_refresh
+   WHERE known_coin_id=coin_uuid;
+
+  IF tmp_val IS NOT NULL
+  THEN
+    spent_val = spent_val + tmp_val;
+    spent_frac = spent_frac + tmp_frac;
+  END IF;
+
+  SELECT
+     SUM(amount_val) -- overflow here is not plausible
+    ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits
+    INTO
+     tmp_val
+    ,tmp_frac
+    FROM recoup
+   WHERE known_coin_id=coin_uuid;
+
+  IF tmp_val IS NOT NULL
+  THEN
+    spent_val = spent_val + tmp_val;
+    spent_frac = spent_frac + tmp_frac;
+  END IF;
+
+  SELECT
+     SUM(amount_val) -- overflow here is not plausible
+    ,SUM(CAST(amount_frac AS INT8)) -- compute using 64 bits
+    INTO
+     tmp_val
+    ,tmp_frac
+    FROM recoup_refresh
+    JOIN refresh_revealed_coins rrc
+        USING (rrc_serial)
+    JOIN refresh_commitments rfc
+         ON (rrc.melt_serial_id = rfc.melt_serial_id)
+   WHERE rfc.old_known_coin_id=coin_uuid;
+
+  IF tmp_val IS NOT NULL
+  THEN
+    unspent_val = unspent_val + tmp_val;
+    unspent_frac = unspent_frac + tmp_frac;
+  END IF;
+
+  IF ( (0 < tmp_val) OR (0 < tmp_frac) )
+  THEN
+    -- There was a transaction that justifies the zombie
+    -- status, clear the flag
+    zombie_required=FALSE;
+  END IF;
+
+END IF;
+
+
+-- normalize results
+spent_val = spent_val + spent_frac / 100000000;
+spent_frac = spent_frac % 100000000;
+unspent_val = unspent_val + unspent_frac / 100000000;
+unspent_frac = unspent_frac % 100000000;
+
+-- Actually check if the coin balance is sufficient. Verbosely. ;-)
+IF (unspent_val > spent_val)
+THEN
+  balance_ok=TRUE;
+ELSE
+  IF (unspent_val = spent_val) AND (unspent_frac >= spent_frac)
+  THEN
+    balance_ok=TRUE;
+  ELSE
+    balance_ok=FALSE;
+  END IF;
+END IF;
+
+zombie_ok = NOT zombie_required;
+
+END $$;
+
+COMMENT ON FUNCTION exchange_do_check_coin_balance(INT8, INT4, BYTEA, BOOLEAN, BOOLEAN)
+  IS 'Checks whether the coin has sufficient balance for all the operations associated with it';
+
+
 
 
 -- Complete transaction

src/exchangedb/plugin_exchangedb_postgres.c

@@ -596,6 +596,16 @@ prepare_statements (struct PostgresClosure *pg)
       "lock_withdraw",
       "LOCK TABLE reserves_out;",
       0),
+    /* Used in #postgres_do_check_coin_balance() to check
+       a coin's balance */
+    GNUNET_PQ_make_prepare (
+      "call_check_coin_balance",
+      "SELECT "
+      " balance_ok"
+      ",zombie_ok"
+      " FROM exchange_do_check_coin_balance"
+      " ($1,$2,$3,$4,$5);",
+      5),
     /* Used in #postgres_do_withdraw() to store
        the signature of a blinded coin with the blinded coin's
        details before returning it during /reserve/withdraw. We store
@@ -798,7 +808,8 @@ prepare_statements (struct PostgresClosure *pg)
       ",noreveal_index "
       ") SELECT $1, known_coin_id, $3, $4, $5, $6"
       "    FROM known_coins"
-      "   WHERE coin_pub=$2",
+      "   WHERE coin_pub=$2"
+      " ON CONFLICT DO NOTHING",
       6),
     /* Used in #postgres_get_melt() to fetch
        high-level information about a melt operation */
@@ -1033,7 +1044,8 @@ prepare_statements (struct PostgresClosure *pg)
       ") SELECT known_coin_id, $2, $3, $4, $5, $6, "
       " $7, $8, $9, $10, $11, $12, $13"
       "    FROM known_coins"
-      "   WHERE coin_pub=$1;",
+      "   WHERE coin_pub=$1"
+      " ON CONFLICT DO NOTHING;",
       13),
     /* Fetch an existing deposit request, used to ensure idempotency
        during /deposit processing. Used in #postgres_have_deposit(). */
@@ -4489,6 +4501,53 @@ postgres_get_withdraw_info (
 }
 
 
+/**
+ * Check coin balance is sufficient to satisfy balance
+ * invariants.
+ *
+ * @param cls the `struct PostgresClosure` with the plugin-specific state
+ * @param coin_pub coin to check
+ * @param coin_value value of the coin's denomination (avoids internal lookup)
+ * @param check_recoup include recoup and recoup_refresh tables in calculation
+ * @param zombie_required additionally require coin to be a zombie coin
+ * @param[out] balance_ok set to true if the balance was sufficient
+ * @param[out] zombie_ok set to true if the zombie requirement was satisfied
+ * @return query execution status
+ */
+static enum GNUNET_DB_QueryStatus
+postgres_do_check_coin_balance (
+  void *cls,
+  const struct TALER_CoinSpendPublicKeyP *coin_pub,
+  const struct TALER_Amount *coin_value,
+  bool check_recoup,
+  bool zombie_required,
+  bool *balance_ok,
+  bool *zombie_ok)
+{
+  struct PostgresClosure *pg = cls;
+  struct GNUNET_PQ_QueryParam params[] = {
+    TALER_PQ_query_param_amount (coin_value),
+    GNUNET_PQ_query_param_auto_from_type (coin_pub),
+    GNUNET_PQ_query_param_bool (check_recoup),
+    GNUNET_PQ_query_param_bool (zombie_required),
+    GNUNET_PQ_query_param_end
+  };
+  struct GNUNET_PQ_ResultSpec rs[] = {
+    GNUNET_PQ_result_spec_bool ("balance_ok",
+                                balance_ok),
+    GNUNET_PQ_result_spec_bool ("zombie_ok",
+                                zombie_ok),
+    GNUNET_PQ_result_spec_end
+  };
+
+  return GNUNET_PQ_eval_prepared_singleton_select (pg->conn,
+                                                   "call_check_coin_balance",
+                                                   params,
+                                                   rs);
+
+}
+
+
 /**
  * Perform withdraw operation, checking for sufficient balance
  * and possibly persisting the withdrawal details.
@@ -5778,6 +5837,8 @@ postgres_ensure_coin_known (void *cls,
     GNUNET_break (0);
     return TALER_EXCHANGEDB_CKS_HARD_FAIL;
   case GNUNET_DB_STATUS_SOFT_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Serialization failure in insert_known_coin? Strange!\n");
     return TALER_EXCHANGEDB_CKS_SOFT_FAIL;
   case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS:
     /* continued below */
@@ -5794,8 +5855,11 @@ postgres_ensure_coin_known (void *cls,
   switch (qs)
   {
   case GNUNET_DB_STATUS_HARD_ERROR:
+    GNUNET_break (0);
     return TALER_EXCHANGEDB_CKS_HARD_FAIL;
   case GNUNET_DB_STATUS_SOFT_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Serialization failure in get_known_coin_dh? Strange!\n");
     return TALER_EXCHANGEDB_CKS_SOFT_FAIL;
   case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT:
     if (0 == GNUNET_memcmp (&denom_pub_hash,
@@ -5865,7 +5929,6 @@ postgres_insert_deposit (void *cls,
                                     &kyc);
   if (qs <= 0)
   {
-    GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs);
     GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
     return qs;
   }
@@ -11819,7 +11882,7 @@ libtaler_plugin_exchangedb_postgres_init (void *cls)
   plugin->get_latest_reserve_in_reference =
     &postgres_get_latest_reserve_in_reference;
   plugin->get_withdraw_info = &postgres_get_withdraw_info;
-  // plugin->insert_withdraw_info = &postgres_insert_withdraw_info;
+  plugin->do_check_coin_balance = &postgres_do_check_coin_balance;
   plugin->do_withdraw = &postgres_do_withdraw;
   plugin->do_withdraw_limit_check = &postgres_do_withdraw_limit_check;
   plugin->get_reserve_history = &postgres_get_reserve_history;

src/include/taler_exchangedb_plugin.h

@@ -2503,18 +2503,27 @@ struct TALER_EXCHANGEDB_Plugin
 
 
   /**
-   * Store collectable coin under the corresponding hash of the blinded
+   * Check coin balance is sufficient to satisfy balance
-   * message.
+   * invariants.
    *
-   * @param cls the @e cls of this struct with the plugin-specific state
+   * @param cls the `struct PostgresClosure` with the plugin-specific state
-   * @param collectable corresponding collectable coin (blind signature)
+   * @param coin_pub coin to check
-   *                    if a coin is found
+   * @param coin_value value of the coin's denomination (avoids internal lookup)
-   * @return statement execution status
+   * @param check_recoup include recoup and recoup_refresh tables in calculation
+   * @param zombie_required additionally require coin to be a zombie coin
+   * @param[out] balance_ok set to true if the balance was sufficient
+   * @param[out] zombie_ok set to true if the zombie requirement was satisfied
+   * @return query execution status
    */
   enum GNUNET_DB_QueryStatus
-  (*insert_withdraw_infoXX)(
+  (*do_check_coin_balance)(
     void *cls,
-    const struct TALER_EXCHANGEDB_CollectableBlindcoin *collectable);
+    const struct TALER_CoinSpendPublicKeyP *coin_pub,
+    const struct TALER_Amount *coin_value,
+    bool check_recoup,
+    bool zombie_required,
+    bool *balance_ok,
+    bool *zombie_ok);
 
 
   /**

src/include/taler_mhd_lib.h

@@ -30,6 +30,12 @@
 #include <gnunet/gnunet_mhd_compat.h>
 
 
+/**
+ * Maximum POST request size.
+ */
+#define TALER_MHD_REQUEST_BUFFER_MAX (1024 * 1024 * 16)
+
+
 /**
  * Global options for response generation.
  */

src/lib/exchange_api_common.c

@@ -386,12 +386,6 @@ TALER_EXCHANGE_parse_reserve_history (
 }
 
 
-/**
- * Free memory (potentially) allocated by #TALER_EXCHANGE_parse_reserve_history().
- *
- * @param rhistory result to free
- * @param len number of entries in @a rhistory
- */
 void
 TALER_EXCHANGE_free_reserve_history (
   struct TALER_EXCHANGE_ReserveHistory *rhistory,
@@ -416,17 +410,6 @@ TALER_EXCHANGE_free_reserve_history (
 }
 
 
-/**
- * Verify a coins transaction history as returned by the exchange.
- *
- * @param dk fee structure for the coin, NULL to skip verifying fees
- * @param currency expected currency for the coin
- * @param coin_pub public key of the coin
- * @param history history of the coin in json encoding
- * @param[out] h_denom_pub set to the hash of the coin's denomination (if available)
- * @param[out] total how much of the coin has been spent according to @a history
- * @return #GNUNET_OK if @a history is valid, #GNUNET_SYSERR if not
- */
 enum GNUNET_GenericReturnValue
 TALER_EXCHANGE_verify_coin_history (
   const struct TALER_EXCHANGE_DenomPublicKey *dk,

src/lib/exchange_api_curl_defaults.c

@@ -1,6 +1,6 @@
 /*
   This file is part of TALER
-  Copyright (C) 2014-2018 Taler Systems SA
+  Copyright (C) 2014-2018, 2021 Taler Systems SA
 
   TALER is free software; you can redistribute it and/or modify it under the
   terms of the GNU General Public License as published by the Free Software
@@ -23,12 +23,6 @@
 #include "exchange_api_curl_defaults.h"
 
 
-/**
- * Get a curl handle with the right defaults for the exchange lib.  In the
- * future, we might manage a pool of connections here.
- *
- * @param url URL to query
- */
 CURL *
 TALER_EXCHANGE_curl_easy_get_ (const char *url)
 {
@@ -45,6 +39,12 @@ TALER_EXCHANGE_curl_easy_get_ (const char *url)
                  curl_easy_setopt (eh,
                                    CURLOPT_FOLLOWLOCATION,
                                    1L));
+  /* Enable compression (using whatever curl likes), see
+     https://curl.se/libcurl/c/CURLOPT_ACCEPT_ENCODING.html  */
+  GNUNET_break (CURLE_OK ==
+                curl_easy_setopt (eh,
+                                  CURLOPT_ACCEPT_ENCODING,
+                                  ""));
   /* limit MAXREDIRS to 5 as a simple security measure against
      a potential infinite loop caused by a malicious target */
   GNUNET_assert (CURLE_OK ==

src/lib/exchange_api_deposit.c

@@ -263,7 +263,7 @@ verify_deposit_signature_conflict (
   ec = TALER_JSON_get_error_code (json);
   switch (ec)
   {
-  case TALER_EC_EXCHANGE_DEPOSIT_INSUFFICIENT_FUNDS:
+  case TALER_EC_EXCHANGE_GENERIC_INSUFFICIENT_FUNDS:
     if (0 >
         TALER_amount_add (&total,
                           &total,

src/lib/exchange_api_handle.c

@@ -52,6 +52,17 @@
  */
 #define EXCHANGE_SERIALIZATION_FORMAT_VERSION 0
 
+/**
+ * How far off do we allow key liftimes to be?
+ */
+#define LIFETIME_TOLERANCE GNUNET_TIME_UNIT_HOURS
+
+/**
+ * If the "Expire" cache control header is missing, for
+ * how long do we assume the reply to be valid at least?
+ */
+#define DEFAULT_EXPIRATION GNUNET_TIME_UNIT_HOURS
+
 /**
  * Set to 1 for extra debug logging.
  */
@@ -155,13 +166,6 @@ struct KeysRequest
 };
 
 
-/**
- * Signature of functions called with the result from our call to the
- * auditor's /deposit-confirmation handler.
- *
- * @param cls closure of type `struct TEAH_AuditorInteractionEntry *`
- * @param hr HTTP response
- */
 void
 TEAH_acc_confirmation_cb (void *cls,
                           const struct TALER_AUDITOR_HttpResponse *hr)
@@ -184,15 +188,6 @@ TEAH_acc_confirmation_cb (void *cls,
 }
 
 
-/**
- * Iterate over all available auditors for @a h, calling
- * @a ac and giving it a chance to start a deposit
- * confirmation interaction.
- *
- * @param h exchange to go over auditors for
- * @param ac function to call per auditor
- * @param ac_cls closure for @a ac
- */
 void
 TEAH_get_auditors_for_dc (struct TALER_EXCHANGE_Handle *h,
                           TEAH_AuditorCallback ac,
@@ -847,7 +842,7 @@ decode_keys_json (const json_t *resp_obj,
       key_data->denom_keys[key_data->num_denom_keys++] = dk;
 
       /* Update "last_denom_issue_date" */
-      TALER_LOG_DEBUG ("Adding denomination key that is valid_from %s\n",
+      TALER_LOG_DEBUG ("Adding denomination key that is valid_until %s\n",
                        GNUNET_STRINGS_absolute_time_to_string (dk.valid_from));
       key_data->last_denom_issue_date
         = GNUNET_TIME_absolute_max (key_data->last_denom_issue_date,
@@ -1043,28 +1038,17 @@ static void
 request_keys (void *cls);
 
 
-/**
- * Let the user set the last valid denomination time manually.
- *
- * @param exchange the exchange handle.
- * @param last_denom_new new last denomination time.
- */
 void
 TALER_EXCHANGE_set_last_denom (struct TALER_EXCHANGE_Handle *exchange,
                                struct GNUNET_TIME_Absolute last_denom_new)
 {
+  TALER_LOG_DEBUG (
+    "Application explicitly set last denomination validity to %s\n",
+    GNUNET_STRINGS_absolute_time_to_string (last_denom_new));
   exchange->key_data.last_denom_issue_date = last_denom_new;
 }
 
 
-/**
- * Check if our current response for /keys is valid, and if
- * not trigger download.
- *
- * @param exchange exchange to check keys for
- * @param flags options controlling when to download what
- * @return until when the response is current, 0 if we are re-downloading
- */
 struct GNUNET_TIME_Absolute
 TALER_EXCHANGE_check_keys_current (struct TALER_EXCHANGE_Handle *exchange,
                                    enum TALER_EXCHANGE_CheckKeysFlags flags)
@@ -1117,9 +1101,19 @@ keys_completed_cb (void *cls,
   };
 
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Received keys from URL `%s' with status %ld.\n",
+              "Received keys from URL `%s' with status %ld and expiration %s.\n",
               kr->url,
-              response_code);
+              response_code,
+              GNUNET_STRINGS_absolute_time_to_string (kr->expire));
+  if (GNUNET_TIME_absolute_is_past (kr->expire))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Exchange failed to give expiration time, assuming in %s\n",
+                GNUNET_STRINGS_relative_time_to_string (DEFAULT_EXPIRATION,
+                                                        GNUNET_YES));
+    kr->expire = GNUNET_TIME_relative_to_absolute (DEFAULT_EXPIRATION);
+    (void) GNUNET_TIME_round_abs (&kr->expire);
+  }
   kd_old = exchange->key_data;
   memset (&kd,
           0,
@@ -1128,6 +1122,9 @@ keys_completed_cb (void *cls,
   switch (response_code)
   {
   case 0:
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Failed to receive /keys response from exchange %s\n",
+                exchange->url);
     free_keys_request (kr);
     exchange->keys_error_count++;
     exchange->kr = NULL;
@@ -1253,9 +1250,14 @@ keys_completed_cb (void *cls,
     break;
   }
   exchange->key_data = kd;
-  TALER_LOG_DEBUG ("Last DK issue date update to: %s\n",
+  if (GNUNET_TIME_absolute_is_past (exchange->key_data.last_denom_issue_date))
-                   GNUNET_STRINGS_absolute_time_to_string
+    TALER_LOG_WARNING ("Last DK issue date from exchange is in the past: %s\n",
-                     (exchange->key_data.last_denom_issue_date));
+                       GNUNET_STRINGS_absolute_time_to_string (
+                         exchange->key_data.last_denom_issue_date));
+  else
+    TALER_LOG_DEBUG ("Last DK issue date updated to: %s\n",
+                     GNUNET_STRINGS_absolute_time_to_string (
+                       exchange->key_data.last_denom_issue_date));
 
 
   if (MHD_HTTP_OK != response_code)
@@ -1298,12 +1300,6 @@ keys_completed_cb (void *cls,
 /* ********************* library internal API ********* */
 
 
-/**
- * Get the context of a exchange.
- *
- * @param h the exchange handle to query
- * @return ctx context to execute jobs in
- */
 struct GNUNET_CURL_Context *
 TEAH_handle_to_context (struct TALER_EXCHANGE_Handle *h)
 {
@@ -1311,12 +1307,6 @@ TEAH_handle_to_context (struct TALER_EXCHANGE_Handle *h)
 }
 
 
-/**
- * Check if the handle is ready to process requests.
- *
- * @param h the exchange handle to query
- * @return #GNUNET_YES if we are ready, #GNUNET_NO if not
- */
 enum GNUNET_GenericReturnValue
 TEAH_handle_is_ready (struct TALER_EXCHANGE_Handle *h)
 {
@@ -1324,13 +1314,6 @@ TEAH_handle_is_ready (struct TALER_EXCHANGE_Handle *h)
 }
 
 
-/**
- * Obtain the URL to use for an API request.
- *
- * @param h handle for the exchange
- * @param path Taler API path (i.e. "/reserve/withdraw")
- * @return the full URL to use with cURL
- */
 char *
 TEAH_path_to_url (struct TALER_EXCHANGE_Handle *h,
                   const char *path)
@@ -1579,17 +1562,6 @@ deserialize_data (struct TALER_EXCHANGE_Handle *exchange,
 }
 
 
-/**
- * Serialize the latest key data from @a
- * exchange to be persisted on disk (to be used with
- * #TALER_EXCHANGE_OPTION_DATA to more efficiently recover
- * the state).
- *
- * @param exchange which exchange's key and wire data should be
- *        serialized
- * @return NULL on error (i.e. no current data available);
- *         otherwise JSON object owned by the caller
- */
 json_t *
 TALER_EXCHANGE_serialize_data (struct TALER_EXCHANGE_Handle *exchange)
 {
@@ -1758,24 +1730,6 @@ TALER_EXCHANGE_serialize_data (struct TALER_EXCHANGE_Handle *exchange)
 }
 
 
-/**
- * Initialise a connection to the exchange. Will connect to the
- * exchange and obtain information about the exchange's master
- * public key and the exchange's auditor.
- * The respective information will be passed to the @a cert_cb
- * once available, and all future interactions with the exchange
- * will be checked to be signed (where appropriate) by the
- * respective master key.
- *
- * @param ctx the context
- * @param url HTTP base URL for the exchange
- * @param cert_cb function to call with the exchange's
- *        certification information
- * @param cert_cb_cls closure for @a cert_cb
- * @param ... list of additional arguments,
- *        terminated by #TALER_EXCHANGE_OPTION_END.
- * @return the exchange handle; NULL upon error
- */
 struct TALER_EXCHANGE_Handle *
 TALER_EXCHANGE_connect (
   struct GNUNET_CURL_Context *ctx,
@@ -1793,7 +1747,7 @@ TALER_EXCHANGE_connect (
   /* Disable 100 continue processing */
   GNUNET_break (GNUNET_OK ==
                 GNUNET_CURL_append_header (ctx,
-                                           "Expect:"));
+                                           MHD_HTTP_HEADER_EXPECT ":"));
   exchange = GNUNET_new (struct TALER_EXCHANGE_Handle);
   exchange->ctx = ctx;
   exchange->url = GNUNET_strdup (url);
@@ -1936,11 +1890,6 @@ request_keys (void *cls)
 }
 
 
-/**
- * Disconnect from the exchange
- *
- * @param exchange the exchange handle
- */
 void
 TALER_EXCHANGE_disconnect (struct TALER_EXCHANGE_Handle *exchange)
 {
@@ -1993,14 +1942,6 @@ TALER_EXCHANGE_disconnect (struct TALER_EXCHANGE_Handle *exchange)
 }
 
 
-/**
- * Test if the given @a pub is a the current signing key from the exchange
- * according to @a keys.
- *
- * @param keys the exchange's key set
- * @param pub claimed current online signing key for the exchange
- * @return #GNUNET_OK if @a pub is (according to /keys) a current signing key
- */
 enum GNUNET_GenericReturnValue
 TALER_EXCHANGE_test_signing_key (const struct TALER_EXCHANGE_Keys *keys,
                                  const struct TALER_ExchangePublicKeyP *pub)
@@ -2010,10 +1951,12 @@ TALER_EXCHANGE_test_signing_key (const struct TALER_EXCHANGE_Keys *keys,
   /* we will check using a tolerance of 1h for the time */
   now = GNUNET_TIME_absolute_get ();
   for (unsigned int i = 0; i<keys->num_sign_keys; i++)
-    if ( (keys->sign_keys[i].valid_from.abs_value_us <= now.abs_value_us + 60
+    if ( (keys->sign_keys[i].valid_from.abs_value_us <=
-          * 60 * 1000LL * 1000LL) &&
+          GNUNET_TIME_absolute_add (now,
-         (keys->sign_keys[i].valid_until.abs_value_us > now.abs_value_us - 60
+                                    LIFETIME_TOLERANCE).abs_value_us) &&
-          * 60 * 1000LL * 1000LL) &&
+         (keys->sign_keys[i].valid_until.abs_value_us >
+          GNUNET_TIME_absolute_subtract (now,
+                                         LIFETIME_TOLERANCE).abs_value_us) &&
          (0 == GNUNET_memcmp (pub,
                               &keys->sign_keys[i].key)) )
       return GNUNET_OK;
@@ -2024,12 +1967,6 @@ TALER_EXCHANGE_test_signing_key (const struct TALER_EXCHANGE_Keys *keys,
 }
 
 
-/**
- * Get exchange's base URL.
- *
- * @param exchange exchange handle.
- * @return the base URL from the handle.
- */
 const char *
 TALER_EXCHANGE_get_base_url (const struct TALER_EXCHANGE_Handle *exchange)
 {
@@ -2037,14 +1974,6 @@ TALER_EXCHANGE_get_base_url (const struct TALER_EXCHANGE_Handle *exchange)
 }
 
 
-/**
- * Obtain the denomination key details from the exchange.
- *
- * @param keys the exchange's key set
- * @param pk public key of the denomination to lookup
- * @return details about the given denomination key, NULL if the key is
- * not found
- */
 const struct TALER_EXCHANGE_DenomPublicKey *
 TALER_EXCHANGE_get_denomination_key (
   const struct TALER_EXCHANGE_Keys *keys,
@@ -2059,12 +1988,6 @@ TALER_EXCHANGE_get_denomination_key (
 }
 
 
-/**
- * Create a copy of a denomination public key.
- *
- * @param key key to copy
- * @returns a copy, must be freed with #TALER_EXCHANGE_destroy_denomination_key
- */
 struct TALER_EXCHANGE_DenomPublicKey *
 TALER_EXCHANGE_copy_denomination_key (
   const struct TALER_EXCHANGE_DenomPublicKey *key)
@@ -2079,12 +2002,6 @@ TALER_EXCHANGE_copy_denomination_key (
 }
 
 
-/**
- * Destroy a denomination public key.
- * Should only be called with keys created by #TALER_EXCHANGE_copy_denomination_key.
- *
- * @param key key to destroy.
- */
 void
 TALER_EXCHANGE_destroy_denomination_key (
   struct TALER_EXCHANGE_DenomPublicKey *key)
@@ -2094,13 +2011,6 @@ TALER_EXCHANGE_destroy_denomination_key (
 }
 
 
-/**
- * Obtain the denomination key details from the exchange.
- *
- * @param keys the exchange's key set
- * @param hc hash of the public key of the denomination to lookup
- * @return details about the given denomination key
- */
 const struct TALER_EXCHANGE_DenomPublicKey *
 TALER_EXCHANGE_get_denomination_key_by_hash (
   const struct TALER_EXCHANGE_Keys *keys,
@@ -2114,12 +2024,6 @@ TALER_EXCHANGE_get_denomination_key_by_hash (
 }
 
 
-/**
- * Obtain the keys from the exchange.
- *
- * @param exchange the exchange handle
- * @return the exchange's key set
- */
 const struct TALER_EXCHANGE_Keys *
 TALER_EXCHANGE_get_keys (struct TALER_EXCHANGE_Handle *exchange)
 {
@@ -2129,13 +2033,6 @@ TALER_EXCHANGE_get_keys (struct TALER_EXCHANGE_Handle *exchange)
 }
 
 
-/**
- * Obtain the keys from the exchange in the
- * raw JSON format
- *
- * @param exchange the exchange handle
- * @return the exchange's keys in raw JSON
- */
 json_t *
 TALER_EXCHANGE_get_keys_raw (struct TALER_EXCHANGE_Handle *exchange)
 {

src/lib/exchange_api_link.c

@@ -418,19 +418,6 @@ handle_link_finished (void *cls,
 }
 
 
-/**
- * Submit a link request to the exchange and get the exchange's response.
- *
- * This API is typically not used by anyone, it is more a threat against those
- * trying to receive a funds transfer by abusing the refresh protocol.
- *
- * @param exchange the exchange handle; the exchange must be ready to operate
- * @param coin_priv private key to request link data for
- * @param link_cb the callback to call with the useful result of the
- *        refresh operation the @a coin_priv was involved in (if any)
- * @param link_cb_cls closure for @a link_cb
- * @return a handle for this request
- */
 struct TALER_EXCHANGE_LinkHandle *
 TALER_EXCHANGE_link (struct TALER_EXCHANGE_Handle *exchange,
                      const struct TALER_CoinSpendPrivateKeyP *coin_priv,
@@ -496,12 +483,6 @@ TALER_EXCHANGE_link (struct TALER_EXCHANGE_Handle *exchange,
 }
 
 
-/**
- * Cancel a link request.  This function cannot be used
- * on a request handle if the callback was already invoked.
- *
- * @param lh the link handle
- */
 void
 TALER_EXCHANGE_link_cancel (struct TALER_EXCHANGE_LinkHandle *lh)
 {

src/lib/exchange_api_management_post_keys.c

@@ -99,6 +99,10 @@ handle_post_keys_finished (void *cls,
     hr.ec = TALER_JSON_get_error_code (json);
     hr.hint = TALER_JSON_get_error_hint (json);
     break;
+  case MHD_HTTP_REQUEST_ENTITY_TOO_LARGE:
+    hr.ec = TALER_JSON_get_error_code (json);
+    hr.hint = TALER_JSON_get_error_hint (json);
+    break;
   default:
     /* unexpected response code */
     GNUNET_break_op (0);

src/lib/exchange_api_melt.c

@@ -97,7 +97,7 @@ struct TALER_EXCHANGE_MeltHandle
  * @param[out] noreveal_index set to the noreveal index selected by the exchange
  * @return #GNUNET_OK if the signature is valid, #GNUNET_SYSERR if not
  */
-static int
+static enum GNUNET_GenericReturnValue
 verify_melt_signature_ok (struct TALER_EXCHANGE_MeltHandle *mh,
                           const json_t *json,
                           struct TALER_ExchangePublicKeyP *exchange_pub,
@@ -208,7 +208,7 @@ verify_melt_signature_denom_conflict (struct TALER_EXCHANGE_MeltHandle *mh,
  * @param json json reply with the signature(s) and transaction history
  * @return #GNUNET_OK if the signature(s) is valid, #GNUNET_SYSERR if not
  */
-static int
+static enum GNUNET_GenericReturnValue
 verify_melt_signature_spend_conflict (struct TALER_EXCHANGE_MeltHandle *mh,
                                       const json_t *json)
 {
@@ -282,7 +282,7 @@ verify_melt_signature_spend_conflict (struct TALER_EXCHANGE_MeltHandle *mh,
   ec = TALER_JSON_get_error_code (json);
   switch (ec)
   {
-  case TALER_EC_EXCHANGE_MELT_INSUFFICIENT_FUNDS:
+  case TALER_EC_EXCHANGE_GENERIC_INSUFFICIENT_FUNDS:
     /* check if melt operation was really too expensive given history */
     if (0 >
         TALER_amount_add (&total,
@@ -379,7 +379,7 @@ handle_melt_finished (void *cls,
     hr.ec = TALER_JSON_get_error_code (j);
     switch (hr.ec)
     {
-    case TALER_EC_EXCHANGE_MELT_INSUFFICIENT_FUNDS:
+    case TALER_EC_EXCHANGE_GENERIC_INSUFFICIENT_FUNDS:
       /* Double spending; check signatures on transaction history */
       if (GNUNET_OK !=
           verify_melt_signature_spend_conflict (mh,

src/mhd/mhd_parsing.c

@@ -27,12 +27,6 @@
 #include "taler_mhd_lib.h"
 
 
-/**
- * Maximum POST request size.
- */
-#define REQUEST_BUFFER_MAX (1024 * 1024)
-
-
 /**
  * Process a POST request containing a JSON object.  This function
  * realizes an MHD POST processor that will (incrementally) process
@@ -65,7 +59,7 @@ TALER_MHD_parse_post_json (struct MHD_Connection *connection,
 {
   enum GNUNET_JSON_PostResult pr;
 
-  pr = GNUNET_JSON_post_parser (REQUEST_BUFFER_MAX,
+  pr = GNUNET_JSON_post_parser (TALER_MHD_REQUEST_BUFFER_MAX,
                                 connection,
                                 con_cls,
                                 upload_data,
@@ -87,9 +81,9 @@ TALER_MHD_parse_post_json (struct MHD_Connection *connection,
     return GNUNET_YES;
   case GNUNET_JSON_PR_REQUEST_TOO_LARGE:
     GNUNET_break (NULL == *json);
-    return (MHD_NO ==
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-            TALER_MHD_reply_request_too_large
+                "Closing connection, upload too large\n");
-              (connection)) ? GNUNET_SYSERR : GNUNET_NO;
+    return MHD_NO;
   case GNUNET_JSON_PR_JSON_INVALID:
     GNUNET_break (NULL == *json);
     return (MHD_YES ==

src/mhd/mhd_responses.c

@@ -419,24 +419,10 @@ TALER_MHD_reply_with_ec (struct MHD_Connection *connection,
 MHD_RESULT
 TALER_MHD_reply_request_too_large (struct MHD_Connection *connection)
 {
-  struct MHD_Response *response;
+  return TALER_MHD_reply_with_error (connection,
-
+                                     MHD_HTTP_REQUEST_ENTITY_TOO_LARGE,
-  response = MHD_create_response_from_buffer (0,
+                                     TALER_EC_GENERIC_UPLOAD_EXCEEDS_LIMIT,
-                                              NULL,
+                                     NULL);
-                                              MHD_RESPMEM_PERSISTENT);
-  if (NULL == response)
-    return MHD_NO;
-  TALER_MHD_add_global_headers (response);
-
-  {
-    MHD_RESULT ret;
-
-    ret = MHD_queue_response (connection,
-                              MHD_HTTP_REQUEST_ENTITY_TOO_LARGE,
-                              response);
-    MHD_destroy_response (response);
-    return ret;
-  }
 }
 
 

src/testing/testing_api_cmd_oauth.c

@@ -79,7 +79,7 @@ append (char **target,
 }
 
 
-static enum MHD_Result
+static MHD_RESULT
 handle_post (void *cls,
              enum MHD_ValueKind kind,
              const char *key,
@@ -160,7 +160,7 @@ handle_post (void *cls,
  *         #MHD_NO if the socket must be closed due to a serious
  *         error while handling the request
  */
-static enum MHD_Result
+static MHD_RESULT
 handler_cb (void *cls,
             struct MHD_Connection *connection,
             const char *url,
@@ -210,7 +210,7 @@ handler_cb (void *cls,
   }
   if (0 != *upload_data_size)
   {
-    enum MHD_Result ret;
+    MHD_RESULT ret;
 
     ret = MHD_post_process (rc->pp,
                             upload_data,

src/util/.gitignore

@@ -1,6 +1,5 @@
 taler-config
 test_payto
-taler-crypto-worker
 taler-exchange-secmod-rsa
 taler-exchange-secmod-eddsa
 test_helper_rsa

src/util/Makefile.am

@@ -58,18 +58,6 @@ taler_exchange_secmod_eddsa_LDADD = \
   $(LIBGCRYPT_LIBS) \
   $(XLIB)
 
-taler_crypto_worker_SOURCES = \
-  taler-crypto-worker.c
-taler_crypto_worker_LDADD = \
-  libtalerutil.la \
-  -lgnunetutil \
-  -lgnunetjson \
-  -ljansson \
-  -lpthread \
-  $(LIBGCRYPT_LIBS) \
-  $(XLIB)
-
-
 lib_LTLIBRARIES = \
   libtalerutil.la
 

src/util/taler-exchange-secmod-rsa.c

@@ -868,6 +868,19 @@ update_keys (struct Denomination *denom,
              bool *wake)
 {
   /* create new denomination keys */
+  if (NULL != denom->keys_tail)
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Updating keys of denomination `%s', last key %s valid for another %s\n",
+                denom->section,
+                GNUNET_h2s (&denom->keys_tail->h_rsa.hash),
+                GNUNET_STRINGS_relative_time_to_string (
+                  GNUNET_TIME_absolute_get_remaining (
+                    GNUNET_TIME_absolute_subtract (
+                      GNUNET_TIME_absolute_add (
+                        denom->keys_tail->anchor,
+                        denom->duration_withdraw),
+                      overlap_duration)),
+                  GNUNET_YES));
   while ( (NULL == denom->keys_tail) ||
           GNUNET_TIME_absolute_is_past (
             GNUNET_TIME_absolute_subtract (

src/util/test_helper_eddsa.conf

@@ -1,7 +1,6 @@
 [PATHS]
 # Persistent data storage for the testcase
 TALER_TEST_HOME = test_helper_eddsa_home/
-TALER_RUNTIME_DIR = ${TMPDIR:-/tmp}/${USER}/test_helper_eddsa/
 
 [taler-exchange-secmod-eddsa]
 CLIENT_DIR = $TALER_RUNTIME_DIR

src/util/test_helper_rsa.c

@@ -609,6 +609,8 @@ main (int argc,
 
   (void) argc;
   (void) argv;
+  unsetenv ("XDG_DATA_HOME");
+  unsetenv ("XDG_CONFIG_HOME");
   GNUNET_log_setup ("test-helper-rsa",
                     "WARNING",
                     NULL);

src/util/test_helper_rsa.conf

@@ -1,8 +1,6 @@
 [PATHS]
 # Persistent data storage for the testcase
 TALER_TEST_HOME = test_helper_rsa_home/
-TALER_RUNTIME_DIR = ${TMPDIR:-/tmp}/${USER}/test_helper_rsa/
-
 
 [coin_1]
 DURATION_WITHDRAW = 1 minute