implement retries for a few more commands

3 files changed, 414 insertions, 52 deletions

diff --git a/src/exchange-lib/testing_api_cmd_deposit.c b/src/exchange-lib/testing_api_cmd_deposit.c
index c07e8fbb..b3f179f9 100644
--- a/src/exchange-lib/testing_api_cmd_deposit.c
+++ b/src/exchange-lib/testing_api_cmd_deposit.c
@@ -82,11 +82,6 @@ struct DepositState
   struct TALER_EXCHANGE_DepositHandle *dh;
 
   /**
-   * Expected HTTP response code.
-   */
-  unsigned int expected_response_code;
-
-  /**
    * Interpreter state.
    */
   struct TALER_TESTING_Interpreter *is;
@@ -95,8 +90,60 @@ struct DepositState
    * Exchange connection.
    */
   struct TALER_EXCHANGE_Handle *exchange;
+
+  /**
+   * Task scheduled to try later.
+   */
+  struct GNUNET_SCHEDULER_Task *retry_task;
+
+  /**
+   * How long do we wait until we retry?
+   */
+  struct GNUNET_TIME_Relative backoff;
+
+  /**
+   * Expected HTTP response code.
+   */
+  unsigned int expected_response_code;
+
+  /**
+   * Should we retry on (transient) failures?
+   */
+  int do_retry;
+
 };
 
+
+/**
+ * Run the command.
+ *
+ * @param cls closure.
+ * @param cmd the command to execute.
+ * @param is the interpreter state.
+ */
+static void
+deposit_run (void *cls,
+             const struct TALER_TESTING_Command *cmd,
+             struct TALER_TESTING_Interpreter *is);
+
+
+/**
+ * Task scheduled to re-try #deposit_run.
+ *
+ * @param cls a `struct DepositState`
+ */
+static void
+do_retry (void *cls)
+{
+  struct DepositState *ds = cls;
+
+  ds->retry_task = NULL;
+  deposit_run (ds,
+               NULL,
+               ds->is);
+}
+
+
 /**
  * Callback to analyze the /deposit response, just used to
  * check if the response code is acceptable.
@@ -120,6 +167,27 @@ deposit_cb (void *cls,
   ds->dh = NULL;
   if (ds->expected_response_code != http_status)
   {
+    if (GNUNET_YES == ds->do_retry)
+    {
+      if ( (0 == http_status) ||
+           (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+	   (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "Retrying deposit failed with %u/%d\n",
+                    http_status,
+                    (int) ec);
+	/* on DB conflicts, do not use backoff */
+	if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
+	  ds->backoff = GNUNET_TIME_UNIT_ZERO;
+	else
+	  ds->backoff = GNUNET_TIME_STD_BACKOFF (ds->backoff);
+	ds->retry_task = GNUNET_SCHEDULER_add_delayed (ds->backoff,
+						       &do_retry,
+						       ds);
+        return;
+      }
+    }
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Unexpected response code %u to command %s in %s:%u\n",
                 http_status,
@@ -324,7 +392,11 @@ deposit_cleanup (void *cls,
     TALER_EXCHANGE_deposit_cancel (ds->dh);
     ds->dh = NULL;
   }
-
+  if (NULL != ds->retry_task)
+  {
+    GNUNET_SCHEDULER_cancel (ds->retry_task);
+    ds->retry_task = NULL;
+  }
   json_decref (ds->wire_details);
   GNUNET_free (ds);
 }
@@ -441,3 +513,25 @@ TALER_TESTING_cmd_deposit
 
   return cmd;
 }
+
+
+/**
+ * Modify a deposit command to enable retries when we get transient
+ * errors from the exchange.
+ *
+ * @param cmd a deposit command
+ * @return the command with retries enabled
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_deposit_with_retry (struct TALER_TESTING_Command cmd)
+{
+  struct DepositState *ds;
+
+  GNUNET_assert (&deposit_run == cmd.run);
+  ds = cmd.cls;
+  ds->do_retry = GNUNET_YES;
+  return cmd;
+}
+
+
+/* end of testing_api_cmd_deposit.c */
diff --git a/src/exchange-lib/testing_api_cmd_refresh.c b/src/exchange-lib/testing_api_cmd_refresh.c
index 0f8e5fc4..788e82a0 100644
--- a/src/exchange-lib/testing_api_cmd_refresh.c
+++ b/src/exchange-lib/testing_api_cmd_refresh.c
@@ -56,19 +56,6 @@ struct RefreshMeltState
 {
 
   /**
-   * if set to GNUNET_YES, then two /refresh/melt operations
-   * will be performed.  This is needed to trigger the logic
-   * that manages those already-made requests.  Note: it
-   * is not possible to just copy-and-paste a test refresh melt
-   * CMD to have the same effect, because every data preparation
-   * generates new planchets that (in turn) make the whole "hash"
-   * different from any previous one, therefore NOT allowing the
-   * exchange to pick any previous /rerfesh/melt operation from
-   * the database.
-   */
-  unsigned int double_melt;
-
-  /**
    * Information about coins to be melted.
    */
   struct MeltDetails melted_coin;
@@ -79,11 +66,6 @@ struct RefreshMeltState
   char *refresh_data;
 
   /**
-   * Number of bytes in @e refresh_data.
-   */
-  size_t refresh_data_length;
-
-  /**
    * Reference to a previous melt command.
    */
   const char *melt_reference;
@@ -104,15 +86,48 @@ struct RefreshMeltState
   struct TALER_TESTING_Interpreter *is;
 
   /**
+   * Array of the denomination public keys
+   * corresponding to the @e fresh_amounts.
+   */
+  struct TALER_EXCHANGE_DenomPublicKey *fresh_pks;
+
+  /**
+   * Task scheduled to try later.
+   */
+  struct GNUNET_SCHEDULER_Task *retry_task;
+
+  /**
+   * How long do we wait until we retry?
+   */
+  struct GNUNET_TIME_Relative backoff;
+
+  /**
+   * Number of bytes in @e refresh_data.
+   */
+  size_t refresh_data_length;
+
+  /**
    * Expected HTTP response code.
    */
   unsigned int expected_response_code;
 
   /**
-   * Array of the denomination public keys
-   * corresponding to the @e fresh_amounts.
+   * if set to #GNUNET_YES, then two /refresh/melt operations
+   * will be performed.  This is needed to trigger the logic
+   * that manages those already-made requests.  Note: it
+   * is not possible to just copy-and-paste a test refresh melt
+   * CMD to have the same effect, because every data preparation
+   * generates new planchets that (in turn) make the whole "hash"
+   * different from any previous one, therefore NOT allowing the
+   * exchange to pick any previous /rerfesh/melt operation from
+   * the database.
    */
-  struct TALER_EXCHANGE_DenomPublicKey *fresh_pks;
+  unsigned int double_melt;
+
+  /**
+   * Should we retry on (transient) failures?
+   */
+  int do_retry;
 
   /**
    * Set by the melt callback as it comes from the exchange.
@@ -137,13 +152,6 @@ struct RefreshRevealState
   struct TALER_EXCHANGE_RefreshRevealHandle *rrh;
 
   /**
-   * Number of fresh coins withdrawn, set by the
-   * reveal callback as it comes from the exchange,
-   * it is the length of the @e fresh_coins array.
-   */
-  unsigned int num_fresh_coins;
-
-  /**
    * Convenience struct to keep in one place all the
    * data related to one fresh coin, set by the reveal callback
    * as it comes from the exchange.
@@ -161,9 +169,32 @@ struct RefreshRevealState
   struct TALER_TESTING_Interpreter *is;
 
   /**
+   * Task scheduled to try later.
+   */
+  struct GNUNET_SCHEDULER_Task *retry_task;
+
+  /**
+   * How long do we wait until we retry?
+   */
+  struct GNUNET_TIME_Relative backoff;
+
+  /**
+   * Number of fresh coins withdrawn, set by the
+   * reveal callback as it comes from the exchange,
+   * it is the length of the @e fresh_coins array.
+   */
+  unsigned int num_fresh_coins;
+
+  /**
    * Expected HTTP response code.
    */
   unsigned int expected_response_code;
+
+  /**
+   * Should we retry on (transient) failures?
+   */
+  int do_retry;
+
 };
 
 
@@ -193,13 +224,59 @@ struct RefreshLinkState
   struct TALER_TESTING_Interpreter *is;
 
   /**
+   * Task scheduled to try later.
+   */
+  struct GNUNET_SCHEDULER_Task *retry_task;
+
+  /**
+   * How long do we wait until we retry?
+   */
+  struct GNUNET_TIME_Relative backoff;
+
+  /**
    * Expected HTTP response code.
    */
   unsigned int expected_response_code;
+
+  /**
+   * Should we retry on (transient) failures?
+   */
+  int do_retry;
+
 };
 
 
 /**
+ * Run the command.
+ *
+ * @param cls closure.
+ * @param cmd the command to execute.
+ * @param is the interpreter state.
+ */
+static void
+refresh_reveal_run (void *cls,
+                    const struct TALER_TESTING_Command *cmd,
+                    struct TALER_TESTING_Interpreter *is);
+
+
+/**
+ * Task scheduled to re-try #refresh_reveal_run.
+ *
+ * @param cls a `struct RefreshRevealState`
+ */
+static void
+do_reveal_retry (void *cls)
+{
+  struct RefreshRevealState *rrs = cls;
+
+  rrs->retry_task = NULL;
+  refresh_reveal_run (rrs,
+                      NULL,
+                      rrs->is);
+}
+
+
+/**
  * "refresh reveal" request callback; it checks that the response
  * code is expected and copies into its command's state the data
  * coming from the exchange, namely the fresh coins.
@@ -231,6 +308,27 @@ reveal_cb (void *cls,
   rrs->rrh = NULL;
   if (rrs->expected_response_code != http_status)
   {
+    if (GNUNET_YES == rrs->do_retry)
+    {
+      if ( (0 == http_status) ||
+           (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+	   (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "Retrying refresh reveal failed with %u/%d\n",
+                    http_status,
+                    (int) ec);
+	/* on DB conflicts, do not use backoff */
+	if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
+	  rrs->backoff = GNUNET_TIME_UNIT_ZERO;
+	else
+	  rrs->backoff = GNUNET_TIME_STD_BACKOFF (rrs->backoff);
+	rrs->retry_task = GNUNET_SCHEDULER_add_delayed (rrs->backoff,
+                                                        &do_reveal_retry,
+                                                        rrs);
+        return;
+      }
+    }
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Unexpected response code %u/%d to command %s in %s:%u\n",
                 http_status,
@@ -258,16 +356,18 @@ reveal_cb (void *cls,
       (num_coins, struct FreshCoin);
 
     const struct TALER_EXCHANGE_DenomPublicKey *fresh_pks;
-    unsigned int i;
-    if (GNUNET_OK != TALER_TESTING_get_trait_denom_pub
-      (melt_cmd, 0, &fresh_pks))
+
+    if (GNUNET_OK !=
+        TALER_TESTING_get_trait_denom_pub (melt_cmd,
+                                           0,
+                                           &fresh_pks))
     {
       GNUNET_break (0);
       TALER_TESTING_interpreter_fail (rrs->is);
       return;
     }
 
-    for (i=0; i<num_coins; i++)
+    for (unsigned int i=0; i<num_coins; i++)
     {
       struct FreshCoin *fc = &rrs->fresh_coins[i];
 
@@ -352,6 +452,11 @@ refresh_reveal_cleanup (void *cls,
     TALER_EXCHANGE_refresh_reveal_cancel (rrs->rrh);
     rrs->rrh = NULL;
   }
+  if (NULL != rrs->retry_task)
+  {
+    GNUNET_SCHEDULER_cancel (rrs->retry_task);
+    rrs->retry_task = NULL;
+  }
 
   for (unsigned int j=0; j < rrs->num_fresh_coins; j++)
     GNUNET_CRYPTO_rsa_signature_free (rrs->fresh_coins[j].sig.rsa_signature);
@@ -363,6 +468,36 @@ refresh_reveal_cleanup (void *cls,
 
 
 /**
+ * Run the command.
+ *
+ * @param cls closure.
+ * @param cmd the command to execute.
+ * @param is the interpreter state.
+ */
+static void
+refresh_link_run (void *cls,
+                  const struct TALER_TESTING_Command *cmd,
+                  struct TALER_TESTING_Interpreter *is);
+
+
+/**
+ * Task scheduled to re-try #refresh_link_run.
+ *
+ * @param cls a `struct RefreshLinkState`
+ */
+static void
+do_link_retry (void *cls)
+{
+  struct RefreshLinkState *rls = cls;
+
+  rls->retry_task = NULL;
+  refresh_link_run (rls,
+                    NULL,
+                    rls->is);
+}
+
+
+/**
  * "refresh link" operation callback, checks that HTTP response
  * code is expected _and_ that all the linked coins were actually
  * withdrawn by the "refresh reveal" CMD.
@@ -402,6 +537,27 @@ link_cb (void *cls,
   rls->rlh = NULL;
   if (rls->expected_response_code != http_status)
   {
+    if (GNUNET_YES == rls->do_retry)
+    {
+      if ( (0 == http_status) ||
+           (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+	   (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "Retrying refresh link failed with %u/%d\n",
+                    http_status,
+                    (int) ec);
+	/* on DB conflicts, do not use backoff */
+	if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
+	  rls->backoff = GNUNET_TIME_UNIT_ZERO;
+	else
+	  rls->backoff = GNUNET_TIME_STD_BACKOFF (rls->backoff);
+	rls->retry_task = GNUNET_SCHEDULER_add_delayed (rls->backoff,
+                                                        &do_link_retry,
+                                                        rls);
+        return;
+      }
+    }
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Unexpected response code %u/%d to command %s in %s:%u\n",
                 http_status,
@@ -514,11 +670,9 @@ refresh_link_run (void *cls,
                   const struct TALER_TESTING_Command *cmd,
                   struct TALER_TESTING_Interpreter *is)
 {
-
   struct RefreshLinkState *rls = cls;
   struct RefreshRevealState *rrs;
   struct RefreshMeltState *rms;
-
   const struct TALER_TESTING_Command *reveal_cmd;
   const struct TALER_TESTING_Command *melt_cmd;
   const struct TALER_TESTING_Command *coin_cmd;
@@ -605,6 +759,41 @@ refresh_link_cleanup (void *cls,
     TALER_EXCHANGE_refresh_link_cancel (rls->rlh);
     rls->rlh = NULL;
   }
+  if (NULL != rls->retry_task)
+  {
+    GNUNET_SCHEDULER_cancel (rls->retry_task);
+    rls->retry_task = NULL;
+  }
+}
+
+
+/**
+ * Run the command.
+ *
+ * @param cls closure.
+ * @param cmd the command to execute.
+ * @param is the interpreter state.
+ */
+static void
+refresh_melt_run (void *cls,
+                  const struct TALER_TESTING_Command *cmd,
+                  struct TALER_TESTING_Interpreter *is);
+
+
+/**
+ * Task scheduled to re-try #refresh_melt_run.
+ *
+ * @param cls a `struct RefreshMeltState`
+ */
+static void
+do_melt_retry (void *cls)
+{
+  struct RefreshMeltState *rms = cls;
+
+  rms->retry_task = NULL;
+  refresh_melt_run (rms,
+                    NULL,
+                    rms->is);
 }
 
 
@@ -634,6 +823,27 @@ melt_cb (void *cls,
   rms->rmh = NULL;
   if (rms->expected_response_code != http_status)
   {
+    if (GNUNET_YES == rms->do_retry)
+    {
+      if ( (0 == http_status) ||
+           (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+	   (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "Retrying refresh melt failed with %u/%d\n",
+                    http_status,
+                    (int) ec);
+	/* on DB conflicts, do not use backoff */
+	if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
+	  rms->backoff = GNUNET_TIME_UNIT_ZERO;
+	else
+	  rms->backoff = GNUNET_TIME_STD_BACKOFF (rms->backoff);
+	rms->retry_task = GNUNET_SCHEDULER_add_delayed (rms->backoff,
+                                                        &do_melt_retry,
+                                                        rms);
+        return;
+      }
+    }
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Unexpected response code %u/%d to command %s in %s:%u\n",
                 http_status,
@@ -668,7 +878,7 @@ melt_cb (void *cls,
  * @param cmd the command to execute.
  * @param is the interpreter state.
  */
-void
+static void
 refresh_melt_run (void *cls,
                   const struct TALER_TESTING_Command *cmd,
                   struct TALER_TESTING_Interpreter *is)
@@ -819,6 +1029,11 @@ refresh_melt_cleanup (void *cls,
     TALER_EXCHANGE_refresh_melt_cancel (rms->rmh);
     rms->rmh = NULL;
   }
+  if (NULL != rms->retry_task)
+  {
+    GNUNET_SCHEDULER_cancel (rms->retry_task);
+    rms->retry_task = NULL;
+  }
   GNUNET_free_non_null (rms->fresh_pks);
   rms->fresh_pks = NULL;
   GNUNET_free_non_null (rms->refresh_data);
@@ -894,10 +1109,10 @@ TALER_TESTING_cmd_refresh_melt
   cmd.run = &refresh_melt_run;
   cmd.cleanup = &refresh_melt_cleanup;
   cmd.traits = &refresh_melt_traits;
-
   return cmd;
 }
 
+
 /**
  * Create a "refresh melt" CMD that does TWO /refresh/melt
  * requests.  This was needed to test the replay of a valid melt
@@ -938,10 +1153,28 @@ TALER_TESTING_cmd_refresh_melt_double
   cmd.run = &refresh_melt_run;
   cmd.cleanup = &refresh_melt_cleanup;
   cmd.traits = &refresh_melt_traits;
+  return cmd;
+}
+
+
+/**
+ * Modify a "refresh melt" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_melt_with_retry (struct TALER_TESTING_Command cmd)
+{
+  struct RefreshMeltState *rms;
 
+  GNUNET_assert (&refresh_melt_run == cmd.run);
+  rms = cmd.cls;
+  rms->do_retry = GNUNET_YES;
   return cmd;
 }
 
+
 /**
  * Offer internal data from a "refresh reveal" CMD.
  *
@@ -960,23 +1193,22 @@ refresh_reveal_traits (void *cls,
 {
   struct RefreshRevealState *rrs = cls;
   unsigned int num_coins = rrs->num_fresh_coins;
-  #define NUM_TRAITS (num_coins * 3) + 3
+#define NUM_TRAITS (num_coins * 3) + 3
   struct TALER_TESTING_Trait traits[NUM_TRAITS];
-  unsigned int i;
 
   /* Making coin privs traits */
-  for (i=0; i<num_coins; i++)
+  for (unsigned int i=0; i<num_coins; i++)
     traits[i] = TALER_TESTING_make_trait_coin_priv
       (i, &rrs->fresh_coins[i].coin_priv);
 
   /* Making denom pubs traits */
-  for (i=0; i<num_coins; i++)
+  for (unsigned int i=0; i<num_coins; i++)
     traits[num_coins + i]
       = TALER_TESTING_make_trait_denom_pub
         (i, rrs->fresh_coins[i].pk);
 
   /* Making denom sigs traits */
-  for (i=0; i<num_coins; i++)
+  for (unsigned int i=0; i<num_coins; i++)
     traits[(num_coins * 2) + i]
       = TALER_TESTING_make_trait_denom_sig
         (i, &rrs->fresh_coins[i].sig);
@@ -998,6 +1230,7 @@ refresh_reveal_traits (void *cls,
                                   index);
 }
 
+
 /**
  * Create a "refresh reveal" command.
  *
@@ -1028,7 +1261,24 @@ TALER_TESTING_cmd_refresh_reveal
   cmd.run = &refresh_reveal_run;
   cmd.cleanup = &refresh_reveal_cleanup;
   cmd.traits = &refresh_reveal_traits;
+  return cmd;
+}
+
+
+/**
+ * Modify a "refresh reveal" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_reveal_with_retry (struct TALER_TESTING_Command cmd)
+{
+  struct RefreshRevealState *rrs;
 
+  GNUNET_assert (&refresh_reveal_run == cmd.run);
+  rrs = cmd.cls;
+  rrs->do_retry = GNUNET_YES;
   return cmd;
 }
 
@@ -1062,6 +1312,23 @@ TALER_TESTING_cmd_refresh_link
   cmd.label = label;
   cmd.run = &refresh_link_run;
   cmd.cleanup = &refresh_link_cleanup;
+  return cmd;
+}
+
+
+/**
+ * Modify a "refresh link" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_link_with_retry (struct TALER_TESTING_Command cmd)
+{
+  struct RefreshLinkState *rls;
 
+  GNUNET_assert (&refresh_link_run == cmd.run);
+  rls = cmd.cls;
+  rls->do_retry = GNUNET_YES;
   return cmd;
 }
diff --git a/src/exchange-lib/testing_api_cmd_withdraw.c b/src/exchange-lib/testing_api_cmd_withdraw.c
index 90a15b80..7aba3ac3 100644
--- a/src/exchange-lib/testing_api_cmd_withdraw.c
+++ b/src/exchange-lib/testing_api_cmd_withdraw.c
@@ -169,15 +169,16 @@ reserve_withdraw_cb (void *cls,
   {
     if (GNUNET_YES == ws->do_retry)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-		  "Retrying withdraw failed with %u/%d\n",
-		  http_status,
-		  (int) ec);
-      if ( (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+      if ( (0 == http_status) ||
+           (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
 	   (TALER_EC_WITHDRAW_INSUFFICIENT_FUNDS == ec) ||
 	   (TALER_EC_WITHDRAW_RESERVE_UNKNOWN == ec) ||
 	   (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
       {
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "Retrying withdraw failed with %u/%d\n",
+                    http_status,
+                    (int) ec);
 	/* on DB conflicts, do not use backoff */
 	if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
 	  ws->backoff = GNUNET_TIME_UNIT_ZERO;