make transactions smaller to try to reduce rollbacks

ChangeLog

@@ -1,3 +1,7 @@
+Sun Aug 19 15:15:48 CEST 2018
+	Increase various limits and rework transaction scopes to
+	improve scalability. -CG
+
 Tue Apr  3 23:29:06 CEST 2018
 	Releasing Taler v0.5.0
 

src/benchmark/exchange_benchmark_home/.config/taler/account-2.json

@@ -1,5 +1,5 @@
 {
   "url": "payto://x-taler-bank/localhost:8082/2",
-  "salt": "MHJ6P3XF2WEC6WA097H8N1MPCT37T7TBHJ3FDTRBPGX0JNQYHJW6D52J0269WS68WG04FMCD5C0E49YEW7R21EXKC7P1TYTJMVKXNZR",
+  "salt": "6TBY14X6YB6D0J2PGKQX0VVV30118QEQG1R3PYP1GG5XWXHST3ZQ1J3R6EQ85HZXS8DFDTRJN6JGBT8KDQFT0KVFPMGXTWHWC7E1518",
-  "master_sig": "ZGZVVR4S9PH9A494B15QSAYCX6NDVF735JN3426T7QQ77VK6QR971TQX71NXHR8N54RGC5GMC49YPK4RSFCJ2Z9GG1CWJ7MAEQSDC08"
+  "master_sig": "CET51EPQGW01BVQMFXQKYW0BMWGQJJT7QTMGNW1FXYC6T6AB1CRK14S5PH5V34JDX7J8JJPVNV3VKXKH4CM1SVRQTFC5QVVDMJ3JM30"
 }

src/exchange/taler-exchange-httpd_db.c

@@ -33,6 +33,43 @@
 #define MAX_TRANSACTION_COMMIT_RETRIES 100
 
 
+/**
+ * Execute database transaction to ensure coin is known. Run the transaction
+ * logic; IF it returns a non-error code, the transaction logic MUST
+ * NOT queue a MHD response.  IF it returns an hard error, the
+ * transaction logic MUST queue a MHD response and set @a mhd_ret.  IF
+ * it returns the soft error code, the function MAY be called again to
+ * retry and MUST not queue a MHD response.
+ *
+ * @param cls a `struct DepositContext`
+ * @param connection MHD request context
+ * @param session database session and transaction to use
+ * @param[out] mhd_ret set to MHD status on error
+ * @return transaction status
+ */
+enum GNUNET_DB_QueryStatus
+TEH_DB_know_coin_transaction (void *cls,
+                              struct MHD_Connection *connection,
+                              struct TALER_EXCHANGEDB_Session *session,
+                              int *mhd_ret)
+{
+  struct TEH_DB_KnowCoinContext *kcc = cls;
+  enum GNUNET_DB_QueryStatus qs;
+
+  qs = TEH_plugin->ensure_coin_known (TEH_plugin->cls,
+                                      session,
+                                      kcc->coin);
+  if (GNUNET_DB_STATUS_HARD_ERROR == qs)
+  {
+    *mhd_ret
+      = TEH_RESPONSE_reply_internal_db_error (connection,
+                                              TALER_EC_DB_COIN_HISTORY_STORE_ERROR);
+    return GNUNET_DB_STATUS_HARD_ERROR;
+  }
+  return qs;
+}
+
+
 /**
  * Run a database transaction for @a connection.
  * Starts a transaction and calls @a cb.  Upon success,

src/exchange/taler-exchange-httpd_db.h

@@ -24,6 +24,45 @@
 #include <microhttpd.h>
 #include "taler_exchangedb_plugin.h"
 
+
+/**
+ * Type of closure for #TEH_DB_know_coin_transaction.
+ */
+struct TEH_DB_KnowCoinContext
+{
+  /**
+   * The coin to make sure it is known.
+   */
+  const struct TALER_CoinPublicInfo *coin;
+
+  /**
+   * MHD connection to queue errors with.
+   */
+  struct MHD_Connection *connection;
+};
+
+
+/**
+ * Execute database transaction to ensure coin is known. Run the transaction
+ * logic; IF it returns a non-error code, the transaction logic MUST
+ * NOT queue a MHD response.  IF it returns an hard error, the
+ * transaction logic MUST queue a MHD response and set @a mhd_ret.  IF
+ * it returns the soft error code, the function MAY be called again to
+ * retry and MUST not queue a MHD response.
+ *
+ * @param cls a `struct DepositContext`
+ * @param connection MHD request context
+ * @param session database session and transaction to use
+ * @param[out] mhd_ret set to MHD status on error
+ * @return transaction status
+ */
+enum GNUNET_DB_QueryStatus
+TEH_DB_know_coin_transaction (void *cls,
+                              struct MHD_Connection *connection,
+                              struct TALER_EXCHANGEDB_Session *session,
+                              int *mhd_ret);
+
+
 /**
  * Function implementing a database transaction.  Runs the transaction
  * logic; IF it returns a non-error code, the transaction logic MUST

src/exchange/taler-exchange-httpd_deposit.c

@@ -144,7 +144,15 @@ deposit_transaction (void *cls,
 				 session,
 				 deposit);
   if (qs < 0)
+  {
+    if (GNUNET_DB_STATUS_HARD_ERROR == qs)
+    {
+      *mhd_ret = TEH_RESPONSE_reply_internal_db_error (connection,
+                                                       TALER_EC_DEPOSIT_HISTORY_DB_ERROR);
+      return GNUNET_DB_STATUS_HARD_ERROR;
+    }
     return qs;
+  }
   if (1 == qs)
   {
     struct TALER_Amount amount_without_fee;
@@ -518,6 +526,22 @@ TEH_DEPOSIT_handler_deposit (struct TEH_RequestHandler *rh,
                                               "deposited amount smaller than depositing fee");
   }
 
+  /* make sure coin is 'known' in database */
+  {
+    struct TEH_DB_KnowCoinContext kcc;
+    int mhd_ret;
+
+    kcc.coin = &deposit.coin;
+    kcc.connection = connection;
+    if (GNUNET_OK !=
+        TEH_DB_run_transaction (connection,
+                                "know coin for deposit",
+                                &mhd_ret,
+                                &TEH_DB_know_coin_transaction,
+                                &kcc))
+      return mhd_ret;
+  }
+
   res = verify_and_execute_deposit (connection,
                                     &deposit);
   GNUNET_JSON_parse_free (spec);

src/exchange/taler-exchange-httpd_payback.c

@@ -391,6 +391,22 @@ verify_and_execute_payback (struct MHD_Connection *connection,
                       &pc.h_blind);
   GNUNET_free (coin_ev);
 
+  /* make sure coin is 'known' in database */
+  {
+    struct TEH_DB_KnowCoinContext kcc;
+    int mhd_ret;
+
+    kcc.coin = coin;
+    kcc.connection = connection;
+    if (GNUNET_OK !=
+        TEH_DB_run_transaction (connection,
+                                "know coin for payback",
+                                &mhd_ret,
+                                &TEH_DB_know_coin_transaction,
+                                &kcc))
+      return mhd_ret;
+  }
+
   pc.coin_sig = coin_sig;
   pc.coin_bks = coin_bks;
   pc.coin = coin;

src/exchange/taler-exchange-httpd_refresh_melt.c

@@ -474,6 +474,22 @@ TEH_REFRESH_handler_refresh_melt (struct TEH_RequestHandler *rh,
     goto cleanup;
   }
 
+  /* make sure coin is 'known' in database */
+  {
+    struct TEH_DB_KnowCoinContext kcc;
+    int mhd_ret;
+
+    kcc.coin = &rmc.refresh_session.coin;
+    kcc.connection = connection;
+    if (GNUNET_OK !=
+        TEH_DB_run_transaction (connection,
+                                "know coin for refresh-melt",
+                                &mhd_ret,
+                                &TEH_DB_know_coin_transaction,
+                                &kcc))
+      return mhd_ret;
+  }
+
   res = handle_refresh_melt (connection,
                              &rmc);
 

src/exchange/test_taler_exchange_aggregator.c

@@ -459,14 +459,21 @@ do_deposit (struct Command *cmd)
         plugin->start (plugin->cls,
                        session,
                        "aggregator-test-1")) ||
-       (GNUNET_OK !=
+       (0 >
+        plugin->ensure_coin_known (plugin->cls,
+                                   session,
+                                   &deposit.coin)) ||
+       (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
         plugin->insert_deposit (plugin->cls,
                                 session,
                                 &deposit)) ||
        (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS !=
         plugin->commit (plugin->cls,
                         session)) )
+  {
+    GNUNET_break (0);
     ret = GNUNET_SYSERR;
+  }
   else
     ret = GNUNET_OK;
   GNUNET_CRYPTO_rsa_signature_free (deposit.coin.denom_sig.rsa_signature);

src/exchangedb/perf_taler_exchangedb_interpreter.c

@@ -1205,6 +1205,10 @@ interpret (struct PERF_TALER_EXCHANGEDB_interpreter_state *state)
 
           deposit_index = state->cmd[state->i].details.insert_deposit.index_deposit;
           deposit = state->cmd[deposit_index].exposed.data.deposit;
+          qs = state->plugin->ensure_coin_known (state->plugin->cls,
+                                                 state->session,
+                                                 &deposit->coin);
+          GNUNET_assert (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs);
           qs = state->plugin->insert_deposit (state->plugin->cls,
                                               state->session,
                                               deposit);
@@ -1434,7 +1438,11 @@ interpret (struct PERF_TALER_EXCHANGEDB_interpreter_state *state)
                                                  &refresh_session.amount_with_fee));
           refresh_session.noreveal_index = 1;
           GNUNET_assert (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT ==
-			 state->plugin->insert_melt (state->session,
+			 state->plugin->ensure_coin_known (state->plugin->cls,
+                                                           state->session,
+                                                           &refresh_session.coin));
+          GNUNET_assert (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT ==
+			 state->plugin->insert_melt (state->plugin->cls,
                                                      state->session,
                                                      &refresh_session));
           state->cmd[state->i].exposed.data.rc = refresh_session.rc;
@@ -1449,7 +1457,7 @@ interpret (struct PERF_TALER_EXCHANGEDB_interpreter_state *state)
 
           hash_index = state->cmd[state->i].details.get_refresh_session.index_hash;
           rc = &state->cmd[hash_index].exposed.data.rc;
-          state->plugin->get_melt (state->session,
+          state->plugin->get_melt (state->plugin->cls,
                                    state->session,
                                    rc,
                                    &refresh);

src/exchangedb/plugin_exchangedb_postgres.c

@@ -3197,15 +3197,16 @@ insert_known_coin (void *cls,
  * @return database transaction status, non-negative on success
  */
 static enum GNUNET_DB_QueryStatus
-ensure_coin_known (struct PostgresClosure *cls,
+postgres_ensure_coin_known (void *cls,
-		   struct TALER_EXCHANGEDB_Session *session,
+                            struct TALER_EXCHANGEDB_Session *session,
-		   const struct TALER_CoinPublicInfo *coin)
+                            const struct TALER_CoinPublicInfo *coin)
 {
+  struct PostgresClosure *pc = cls;
   enum GNUNET_DB_QueryStatus qs;
   struct TALER_CoinPublicInfo known_coin;
 
   /* check if the coin is already known */
-  qs = get_known_coin (cls,
+  qs = get_known_coin (pc,
 		       session,
 		       &coin->coin_pub,
 		       &known_coin);
@@ -3222,7 +3223,7 @@ ensure_coin_known (struct PostgresClosure *cls,
   }
   GNUNET_assert (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs);
   /* if not known, insert it */
-  qs = insert_known_coin (cls,
+  qs = insert_known_coin (pc,
 			  session,
 			  coin);
   if (0 >= qs)
@@ -3249,7 +3250,6 @@ postgres_insert_deposit (void *cls,
                          struct TALER_EXCHANGEDB_Session *session,
                          const struct TALER_EXCHANGEDB_Deposit *deposit)
 {
-  enum GNUNET_DB_QueryStatus qs;
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_auto_from_type (&deposit->coin.coin_pub),
     TALER_PQ_query_param_amount (&deposit->amount_with_fee),
@@ -3264,10 +3264,14 @@ postgres_insert_deposit (void *cls,
     GNUNET_PQ_query_param_end
   };
 
-  if (0 > (qs = ensure_coin_known (cls,
+#if 0
-				   session,
+  enum GNUNET_DB_QueryStatus qs;
-				   &deposit->coin)))
+
+  if (0 > (qs = postgres_ensure_coin_known (cls,
+                                            session,
+                                            &deposit->coin)))
     return qs;
+#endif
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
 	      "Inserting deposit to be executed at %s (%llu/%llu)\n",
 	      GNUNET_STRINGS_absolute_time_to_string (deposit->wire_deadline),
@@ -3501,12 +3505,14 @@ postgres_insert_melt (void *cls,
     GNUNET_PQ_query_param_uint32 (&refresh_session->noreveal_index),
     GNUNET_PQ_query_param_end
   };
+#if 0
   enum GNUNET_DB_QueryStatus qs;
 
-  if (0 > (qs = ensure_coin_known (cls,
+  if (0 > (qs = postgres_ensure_coin_known (cls,
-				   session,
+                                            session,
-				   &refresh_session->coin)))
+                                            &refresh_session->coin)))
     return qs;
+#endif
   return GNUNET_PQ_eval_prepared_non_select (session->conn,
 					     "insert_melt",
 					     params);
@@ -6371,11 +6377,13 @@ postgres_insert_payback_request (void *cls,
   };
   enum GNUNET_DB_QueryStatus qs;
 
+#if 0
   /* check if the coin is already known */
-  if (0 > (qs = ensure_coin_known (cls,
+  if (0 > (qs = postgres_ensure_coin_known (cls,
-				   session,
+                                            session,
-				   coin)))
+                                            coin)))
     return qs;
+#endif
   /* now store actual payback information */
   qs = GNUNET_PQ_eval_prepared_non_select (session->conn,
 					   "payback_insert",
@@ -6993,6 +7001,7 @@ libtaler_plugin_exchangedb_postgres_init (void *cls)
   plugin->insert_withdraw_info = &postgres_insert_withdraw_info;
   plugin->get_reserve_history = &postgres_get_reserve_history;
   plugin->free_reserve_history = &common_free_reserve_history;
+  plugin->ensure_coin_known = &postgres_ensure_coin_known;
   plugin->have_deposit = &postgres_have_deposit;
   plugin->mark_deposit_tiny = &postgres_mark_deposit_tiny;
   plugin->test_deposit_done = &postgres_test_deposit_done;

src/exchangedb/test_exchangedb.c

@@ -590,6 +590,10 @@ test_melting (struct TALER_EXCHANGEDB_Session *session)
                             session,
                             &refresh_session.rc,
                             &ret_refresh_session));
+  FAILIF (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
+          plugin->ensure_coin_known (plugin->cls,
+                                     session,
+                                     &refresh_session.coin));
   FAILIF (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
 	  plugin->insert_melt (plugin->cls,
                                session,
@@ -1775,6 +1779,10 @@ run (void *cls)
   deposit.coin.denom_sig = cbc.sig;
   deadline = GNUNET_TIME_absolute_get ();
   (void) GNUNET_TIME_round_abs (&deadline);
+  FAILIF (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
+          plugin->ensure_coin_known (plugin->cls,
+                                     session,
+                                     &deposit.coin));
   FAILIF (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
           plugin->insert_payback_request (plugin->cls,
                                           session,
@@ -1922,6 +1930,10 @@ run (void *cls)
   deposit.refund_deadline = deadline;
   deposit.wire_deadline = deadline;
   result = 8;
+  FAILIF (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
+          plugin->ensure_coin_known (plugin->cls,
+                                     session,
+                                     &deposit.coin));
   FAILIF (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
           plugin->insert_deposit (plugin->cls,
                                   session,

src/include/taler_error_codes.h

@@ -175,6 +175,11 @@ enum TALER_ErrorCode
    */
   TALER_EC_PAYTO_MALFORMED = 1013,
 
+  /**
+   * We failed to update the database of known coins.
+   */
+  TALER_EC_DB_COIN_HISTORY_STORE_ERROR = 1014,
+
   /* ********** request-specific error codes ************* */
 
   /**

src/include/taler_exchangedb_plugin.h

@@ -1397,6 +1397,20 @@ struct TALER_EXCHANGEDB_Plugin
                            struct TALER_EXCHANGEDB_ReserveHistory *rh);
 
 
+  /**
+   * Make sure the given @a coin is known to the database.
+   *
+   * @param cls database connection plugin state
+   * @param session database session
+   * @param coin the coin that must be made known
+   * @return database transaction status, non-negative on success
+   */
+  enum GNUNET_DB_QueryStatus
+  (*ensure_coin_known) (void *cls,
+                        struct TALER_EXCHANGEDB_Session *session,
+                        const struct TALER_CoinPublicInfo *coin);
+
+
   /**
    * Check if we have the specified deposit already in the database.
    *