finished implementing #4956 in principle, but not yet tested

src/auditor/taler-auditor.c

@@ -99,6 +99,11 @@ static struct TALER_AUDITORDB_Plugin *adb;
  */
 static struct TALER_AUDITORDB_Session *asession;
 
+/**
+ * After how long should idle reserves be closed?
+ */
+static struct GNUNET_TIME_Relative idle_reserve_expiration_time;
+
 /**
  * Master public key of the exchange to audit.
  */
@@ -672,7 +677,7 @@ handle_reserve_in (void *cls,
               TALER_B2S (reserve_pub),
               TALER_amount2s (credit));
   expiry = GNUNET_TIME_absolute_add (execution_date,
-                                     TALER_IDLE_RESERVE_EXPIRATION_TIME);
+                                     idle_reserve_expiration_time);
   rs->a_expiration_date = GNUNET_TIME_absolute_max (rs->a_expiration_date,
                                                     expiry);
   return GNUNET_OK;
@@ -973,7 +978,7 @@ handle_payback_by_reserve (void *cls,
               TALER_B2S (reserve_pub),
               TALER_amount2s (amount));
   expiry = GNUNET_TIME_absolute_add (timestamp,
-                                     TALER_IDLE_RESERVE_EXPIRATION_TIME);
+                                     idle_reserve_expiration_time);
   rs->a_expiration_date = GNUNET_TIME_absolute_max (rs->a_expiration_date,
                                                     expiry);
   return GNUNET_OK;
@@ -1002,7 +1007,7 @@ handle_reserve_closed (void *cls,
 		       const struct TALER_Amount *closing_fee,
 		       const struct TALER_ReservePublicKeyP *reserve_pub,
 		       const json_t *receiver_account,
-		       const json_t *transfer_details)
+		       const struct TALER_WireTransferIdentifierRawP *transfer_details)
 {
   struct ReserveContext *rc = cls;
   struct GNUNET_HashCode key;
@@ -3613,6 +3618,18 @@ run (void *cls,
     global_ret = 1;
     return;
   }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (cfg,
+					   "exchangedb",
+					   "IDLE_RESERVE_EXPIRATION_TIME",
+					   &idle_reserve_expiration_time))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               "exchangedb",
+                               "IDLE_RESERVE_EXPIRATION_TIME");
+    global_ret = 1;
+    return;
+  }
   if (NULL ==
       (edb = TALER_EXCHANGEDB_plugin_load (cfg)))
   {

src/exchange-lib/exchange_api_reserve.c

@@ -335,8 +335,8 @@ parse_reserve_history (struct TALER_EXCHANGE_Handle *exchange,
       struct GNUNET_JSON_Specification closing_spec[] = {
         GNUNET_JSON_spec_json ("receiver_account_details",
 			       &rhistory[off].details.close_details.receiver_account_details),
-        GNUNET_JSON_spec_json ("wire_transfer",
+        GNUNET_JSON_spec_fixed_auto ("wire_transfer",
-			       &rhistory[off].details.close_details.transfer_details),
+				     &rhistory[off].details.close_details.wtid),
         GNUNET_JSON_spec_fixed_auto ("exchange_sig",
                                      &rhistory[off].details.close_details.exchange_sig),
         GNUNET_JSON_spec_fixed_auto ("exchange_pub",
@@ -362,8 +362,7 @@ parse_reserve_history (struct TALER_EXCHANGE_Handle *exchange,
 			 &amount);
       TALER_JSON_hash (rhistory[off].details.close_details.receiver_account_details,
 		       &rcc.h_wire);
-      TALER_JSON_hash (rhistory[off].details.close_details.receiver_account_details,
+      rcc.wtid = rhistory[off].details.close_details.wtid;
-		       &rcc.h_transfer);
       rcc.purpose.size = htonl (sizeof (rcc));
       rcc.purpose.purpose = htonl (TALER_SIGNATURE_EXCHANGE_RESERVE_CLOSED);
       rcc.reserve_pub = *reserve_pub;

src/exchange/taler-exchange-aggregator.c

@@ -177,6 +177,34 @@ struct AggregationUnit
 };
 
 
+/**
+ * Context we use while closing a reserve.
+ */
+struct CloseTransferContext
+{
+  /**
+   * Handle for preparing the wire transfer.
+   */
+  struct TALER_WIRE_PrepareHandle *ph;
+
+  /**
+   * Our database session.
+   */
+  struct TALER_EXCHANGEDB_Session *session;
+
+  /**
+   * Wire transfer method.
+   */
+  char *type;
+};
+
+
+/**
+ * Active context while processing reserve closing,
+ * or NULL.
+ */
+static struct CloseTransferContext *ctc;
+
 /**
  * Which currency is used by this exchange?
  */
@@ -235,6 +263,11 @@ static int global_ret;
  */
 static int test_mode;
 
+/**
+ * Did #run_reserve_closures() have any work during its last run?
+ */
+static int reserves_idle;
+
 /**
  * Limit on the number of transactions we aggregate at once.  Note
  * that the limit must be big enough to ensure that when transactions
@@ -666,7 +699,8 @@ aggregate_cb (void *cls,
 
 
 /**
- * Function to be called with the prepared transfer data.
+ * Function to be called with the prepared transfer data
+ * when running an aggregation on a merchant.
  *
  * @param cls closure with the `struct AggregationUnit`
  * @param buf transaction data to persist, NULL on error
@@ -678,6 +712,319 @@ prepare_cb (void *cls,
             size_t buf_size);
 
 
+/**
+ * Main work function that finds and triggers transfers for reserves
+ * closures.
+ *
+ * @param cls closure
+ */
+static void
+run_reserve_closures (void *cls);
+
+
+/**
+ * Main work function that queries the DB and aggregates transactions
+ * into larger wire transfers.
+ *
+ * @param cls NULL
+ */
+static void
+run_aggregation (void *cls);
+
+
+/**
+ * Function to be called with the prepared transfer data
+ * when closing a reserve.
+ *
+ * @param cls closure with a `struct CloseTransferContext`
+ * @param buf transaction data to persist, NULL on error
+ * @param buf_size number of bytes in @a buf, 0 on error
+ */
+static void
+prepare_close_cb (void *cls,
+		  const char *buf,
+		  size_t buf_size)
+{
+  GNUNET_assert (cls == ctc);  
+  ctc->ph = NULL;
+  if (NULL == buf)
+  {
+    GNUNET_break (0); /* why? how to best recover? */
+    db_plugin->rollback (db_plugin->cls,
+                         ctc->session);
+    /* start again */
+    GNUNET_free (ctc->type);
+    GNUNET_free (ctc);
+    ctc = NULL;
+    task = GNUNET_SCHEDULER_add_now (&run_aggregation,
+                                     NULL);
+    return;
+  }
+
+  /* Commit our intention to execute the wire transfer! */
+  if (GNUNET_OK !=
+      db_plugin->wire_prepare_data_insert (db_plugin->cls,
+                                           ctc->session,
+                                           ctc->type,
+                                           buf,
+                                           buf_size))
+  {
+    GNUNET_break (0); /* why? how to best recover? */
+    db_plugin->rollback (db_plugin->cls,
+                         ctc->session);
+    /* start again */
+    task = GNUNET_SCHEDULER_add_now (&run_aggregation,
+                                     NULL);
+    GNUNET_free (ctc->type);
+    GNUNET_free (ctc);
+    ctc = NULL;
+    return;
+  }
+
+  /* finally commit */
+  if (GNUNET_OK !=
+      db_plugin->commit (db_plugin->cls,
+			 ctc->session))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+		"Failed to commit database transaction!\n");
+  }
+  GNUNET_free (ctc->type);
+  GNUNET_free (ctc);
+  ctc = NULL;
+  task = GNUNET_SCHEDULER_add_now (&run_aggregation,
+				   NULL);
+}
+
+
+/**
+ * Function called with details about expired reserves.
+ * We trigger the reserve closure by inserting the respective
+ * closing record and prewire instructions into the respective
+ * tables. 
+ *
+ * @param cls a `struct TALER_EXCHANGEDB_Session *`
+ * @param reserve_pub public key of the reserve
+ * @param left amount left in the reserve
+ * @param account_details information about the reserve's bank account
+ * @param expiration_date when did the reserve expire
+ * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop
+ */
+static int
+expired_reserve_cb (void *cls,
+		    const struct TALER_ReservePublicKeyP *reserve_pub,
+		    const struct TALER_Amount *left,
+		    const json_t *account_details,
+		    struct GNUNET_TIME_Absolute expiration_date)
+{
+  struct TALER_EXCHANGEDB_Session *session = cls;
+  struct GNUNET_TIME_Absolute now;
+  struct TALER_WireTransferIdentifierRawP wtid;
+  struct TALER_Amount amount_without_fee;
+  const struct TALER_Amount *closing_fee;
+  int ret;
+  int iret;
+  const char *type;
+  struct WirePlugin *wp;
+
+  GNUNET_assert (NULL == ctc);
+  now = GNUNET_TIME_absolute_get ();
+
+  /* lookup wire plugin */
+  type = extract_type (account_details);
+  if (NULL == type)
+  {
+    GNUNET_break (0);
+    db_plugin->rollback (db_plugin->cls,
+			 session);
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return GNUNET_SYSERR;
+  }
+  wp = find_plugin (type);
+  if (NULL == wp)
+  {
+    GNUNET_break (0);
+    db_plugin->rollback (db_plugin->cls,
+			 session);
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return GNUNET_SYSERR;
+  }
+  
+  /* lookup `closing_fee` */
+  if (GNUNET_OK !=
+      update_fees (wp,
+		   now,
+		   session))
+  {
+    GNUNET_break (0);
+    db_plugin->rollback (db_plugin->cls,
+			 session);
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return GNUNET_SYSERR;
+  }
+  closing_fee = &wp->af->closing_fee;
+
+  /* calculate transfer amount */
+  ret = TALER_amount_subtract (&amount_without_fee,
+			       left,
+			       closing_fee);
+  if ( (GNUNET_SYSERR == ret) ||
+       (GNUNET_NO == ret) )
+  {
+    /* Closing fee higher than remaining balance, close
+       without wire transfer. */
+    closing_fee = left;
+    TALER_amount_get_zero (left->currency,
+			   &amount_without_fee);
+  }
+
+  /* NOTE: sizeof (*reserve_pub) == sizeof (wtid) right now, but to
+     be future-compatible, we use the memset + min construction */
+  memset (&wtid,
+	  0,
+	  sizeof (wtid));
+  memcpy (&wtid,
+	  reserve_pub,
+	  GNUNET_MIN (sizeof (wtid),
+		      sizeof (*reserve_pub)));
+  iret = db_plugin->insert_reserve_closed (db_plugin->cls,
+					   session,
+					   reserve_pub,
+					   now,
+					   account_details,
+					   &wtid,
+					   left,
+					   closing_fee);
+  if ( (GNUNET_OK == ret) &&
+       (GNUNET_OK == iret) )
+  {
+    /* success, perform wire transfer */
+    if (GNUNET_SYSERR ==
+	wp->wire_plugin->amount_round (wp->wire_plugin->cls,
+				       &amount_without_fee))
+    {
+      GNUNET_break (0);
+      db_plugin->rollback (db_plugin->cls,
+			   session);
+      global_ret = GNUNET_SYSERR;
+      GNUNET_SCHEDULER_shutdown ();
+      return GNUNET_SYSERR;
+    }
+    ctc = GNUNET_new (struct CloseTransferContext);
+    ctc->session = session;
+    ctc->type = GNUNET_strdup (type);
+    ctc->ph
+      = wp->wire_plugin->prepare_wire_transfer (wp->wire_plugin->cls,
+						au->wire,
+						&amount_without_fee,
+						exchange_base_url,
+						&wtid,
+						&prepare_close_cb,
+						ctc);
+    if (NULL == ctc->ph)
+    {
+      GNUNET_break (0);
+      db_plugin->rollback (db_plugin->cls,
+			   session);
+      global_ret = GNUNET_SYSERR;
+      GNUNET_SCHEDULER_shutdown ();
+      GNUNET_free (ctc->type);
+      GNUNET_free (ctc);
+      ctc = NULL;
+    }
+    return GNUNET_SYSERR;
+  }
+  /* Check for hard failure */
+  if (GNUNET_SYSERR == iret)
+  {
+    GNUNET_break (0);
+    db_plugin->rollback (db_plugin->cls,
+                         session);
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return GNUNET_SYSERR;
+  }
+  /* Reserve balance was almost zero; just commit */
+  if (GNUNET_OK !=
+      db_plugin->commit (db_plugin->cls,
+			 session))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+		"Failed to commit database transaction!\n");
+  }
+  task = GNUNET_SCHEDULER_add_now (&run_reserve_closures,
+				   NULL);
+  return GNUNET_SYSERR;
+}
+
+
+/**
+ * Main work function that finds and triggers transfers for reserves
+ * closures.
+ *
+ * @param cls closure
+ */
+static void
+run_reserve_closures (void *cls)
+{
+  struct TALER_EXCHANGEDB_Session *session;
+  int ret;
+  const struct GNUNET_SCHEDULER_TaskContext *tc;
+  
+  task = NULL;
+  reserves_idle = GNUNET_NO;
+  tc = GNUNET_SCHEDULER_get_task_context ();
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Checking for reserves to close\n");
+  if (NULL == (session = db_plugin->get_session (db_plugin->cls)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to obtain database session!\n");
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  if (GNUNET_OK !=
+      db_plugin->start (db_plugin->cls,
+			session))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to start database transaction!\n");
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  ret = db_plugin->get_expired_reserves (db_plugin->cls,
+					 session,
+					 GNUNET_TIME_absolute_get (),
+					 &expired_reserve_cb,
+					 session);
+  if (GNUNET_SYSERR == ret)
+  {
+    GNUNET_break (0);
+    db_plugin->rollback (db_plugin->cls,
+                         session);
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  if (GNUNET_NO == ret)
+  {
+    reserves_idle = GNUNET_YES;
+    db_plugin->rollback (db_plugin->cls,
+                         session);
+    task = GNUNET_SCHEDULER_add_now (&run_aggregation,
+				     NULL);
+    return;
+  }
+}
+
+
 /**
  * Main work function that queries the DB and aggregates transactions
  * into larger wire transfers.
@@ -687,6 +1034,7 @@ prepare_cb (void *cls,
 static void
 run_aggregation (void *cls)
 {
+  static int swap;
   struct TALER_EXCHANGEDB_Session *session;
   unsigned int i;
   int ret;
@@ -696,6 +1044,12 @@ run_aggregation (void *cls)
   tc = GNUNET_SCHEDULER_get_task_context ();
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
     return;
+ if (0 == (++swap % 2))
+  {
+    task = GNUNET_SCHEDULER_add_now (&run_reserve_closures,
+				     NULL);
+    return;
+  }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Checking for ready deposits to aggregate\n");
   if (NULL == (session = db_plugin->get_session (db_plugin->cls)))
@@ -748,6 +1102,10 @@ run_aggregation (void *cls)
     else
     {
       /* nothing to do, sleep for a minute and try again */
+      if (GNUNET_NO == reserves_idle)
+	task = GNUNET_SCHEDULER_add_now (&run_reserve_closures,
+					 NULL);
+      else
 	task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
 					     &run_aggregation,
 					     NULL);

src/exchange/taler-exchange-httpd_responses.c

@@ -886,17 +886,16 @@ compile_reserve_history (const struct TALER_EXCHANGEDB_ReserveHistory *rh,
 	rcc.reserve_pub = pos->details.closing->reserve_pub;
 	TALER_JSON_hash (pos->details.closing->receiver_account_details,
 			 &rcc.h_wire);
-	TALER_JSON_hash (pos->details.closing->transfer_details,
+	rcc.wtid = pos->details.closing->transfer_details;
-			 &rcc.h_transfer);
 	TEH_KS_sign (&rcc.purpose,
 		     &pub,
 		     &sig);
 	GNUNET_assert (0 ==
 		       json_array_append_new (json_history,
-					      json_pack ("{s:s, s:O, s:O, s:o, s:o, s:o, s:o, s:o}",
+					      json_pack ("{s:s, s:O, s:o, s:o, s:o, s:o, s:o, s:o}",
 							 "type", "CLOSING",
 							 "receiver_account_details", pos->details.closing->receiver_account_details,
-							 "transfer_details", pos->details.closing->transfer_details,
+							 "transfer_details", GNUNET_JSON_from_data_auto (&pos->details.closing->transfer_details),
 							 "exchange_pub", GNUNET_JSON_from_data_auto (&pub),
 							 "exchange_sig", GNUNET_JSON_from_data_auto (&sig),
 							 "timestamp", GNUNET_JSON_from_time_abs (pos->details.closing->execution_date),

src/exchange/test-taler-exchange-aggregator-postgres.conf

@@ -19,6 +19,15 @@ MASTER_PUBLIC_KEY = 98NJW3CQHZQGQXTY3K85K531XKPAPAVV4Q5V8PYYRR00NJGZWNVG
 # Expected base URL of the exchange.
 BASE_URL = "https://exchange.taler.net/"
 
+[exchangedb]
+# After how long do we close idle reserves?  The exchange
+# and the auditor must agree on this value.  We currently
+# expect it to be globally defined for the whole system,
+# as there is no way for wallets to query this value.  Thus,
+# it is only configurable for testing, and should be treated
+# as constant in production.
+IDLE_RESERVE_EXPIRATION_TIME = 4 weeks
+
 [exchangedb-postgres]
 
 #The connection string the plugin has to use for connecting to the database

src/exchange/test_taler_exchange_httpd.conf

@@ -19,6 +19,16 @@ MASTER_PUBLIC_KEY = 98NJW3CQHZQGQXTY3K85K531XKPAPAVV4Q5V8PYYRR00NJGZWNVG
 DB = postgres
 
 
+[exchangedb]
+# After how long do we close idle reserves?  The exchange
+# and the auditor must agree on this value.  We currently
+# expect it to be globally defined for the whole system,
+# as there is no way for wallets to query this value.  Thus,
+# it is only configurable for testing, and should be treated
+# as constant in production.
+IDLE_RESERVE_EXPIRATION_TIME = 4 weeks
+
+
 [exchangedb-postgres]
 DB_CONN_STR = "postgres:///talercheck"
 

src/exchangedb/exchangedb.conf

@@ -12,3 +12,12 @@ AUDITOR_BASE_DIR = ${TALER_DATA_HOME}/auditors/
 # contain files "$METHOD.fee" with the cost structure, where
 # $METHOD corresponds to a wire transfer method.
 WIREFEE_BASE_DIR = ${TALER_DATA_HOME}/exchange/wirefees/
+
+
+# After how long do we close idle reserves?  The exchange
+# and the auditor must agree on this value.  We currently
+# expect it to be globally defined for the whole system,
+# as there is no way for wallets to query this value.  Thus,
+# it is only configurable for testing, and should be treated
+# as constant in production.
+IDLE_RESERVE_EXPIRATION_TIME = 4 weeks

src/exchangedb/plugin_exchangedb_common.c

@@ -64,8 +64,6 @@ common_free_reserve_history (void *cls,
       closing = rh->details.closing;
       if (NULL != closing->receiver_account_details)
         json_decref (closing->receiver_account_details);
-      if (NULL != closing->transfer_details)
-        json_decref (closing->transfer_details);
       GNUNET_free (closing);
       break;
     }

src/exchangedb/plugin_exchangedb_postgres.c

@@ -141,6 +141,11 @@ struct PostgresClosure
    * the configuration.
    */
   char *connection_cfg_str;
+
+  /**
+   * After how long should idle reserves be closed?
+   */
+  struct GNUNET_TIME_Relative idle_reserve_expiration_time;
 };
 
 
@@ -316,9 +321,6 @@ postgres_create_tables (void *cls)
            ",fee_refund_curr VARCHAR("TALER_CURRENCY_LEN_STR") NOT NULL"
            ")");
   /* denomination_revocations table is for remembering which denomination keys have been revoked */
-  /* TODO (#4981): change denom_pub_hash to REFERENCE 'denominations', and
-     add denom_pub_hash column to denominations, changing other REFERENCEs
-     also to the hash!? */
   SQLEXEC ("CREATE TABLE IF NOT EXISTS denomination_revocations"
            "(denom_revocations_serial_id BIGSERIAL"
 	   ",denom_pub_hash BYTEA PRIMARY KEY REFERENCES denominations (denom_pub_hash) ON DELETE CASCADE"
@@ -332,6 +334,7 @@ postgres_create_tables (void *cls)
      grabbing the money, depending on the Exchange's terms of service) */
   SQLEXEC ("CREATE TABLE IF NOT EXISTS reserves"
            "(reserve_pub BYTEA PRIMARY KEY CHECK(LENGTH(reserve_pub)=32)"
+	   ",account_details TEXT NOT NULL "
            ",current_balance_val INT8 NOT NULL"
            ",current_balance_frac INT4 NOT NULL"
            ",current_balance_curr VARCHAR("TALER_CURRENCY_LEN_STR") NOT NULL"
@@ -367,7 +370,7 @@ postgres_create_tables (void *cls)
           "(close_uuid BIGSERIAL PRIMARY KEY"
           ",reserve_pub BYTEA NOT NULL REFERENCES reserves (reserve_pub) ON DELETE CASCADE"
 	  ",execution_date INT8 NOT NULL"
-          ",transfer_details TEXT NOT NULL"
+	  ",transfer_details BYTEA NOT NULL CHECK (LENGTH(transfer_details)=32)"
           ",receiver_account TEXT NOT NULL"
           ",amount_val INT8 NOT NULL"
           ",amount_frac INT4 NOT NULL"
@@ -715,13 +718,14 @@ postgres_prepare (PGconn *db_conn)
   PREPARE ("reserve_create",
            "INSERT INTO reserves "
            "(reserve_pub"
+	   ",account_details"
            ",current_balance_val"
            ",current_balance_frac"
            ",current_balance_curr"
            ",expiration_date"
            ") VALUES "
-           "($1, $2, $3, $4, $5);",
+           "($1, $2, $3, $4, $5, $6);",
-           5, NULL);
+           6, NULL);
 
   /* Used in #postgres_insert_reserve_closed() */
   PREPARE ("reserves_close_insert",
@@ -1582,6 +1586,21 @@ postgres_prepare (PGconn *db_conn)
            " WHERE reserve_pub=$1;",
            1, NULL);
 
+  /* Used in #postgres_get_expired_reserves() */
+  PREPARE ("get_expired_reserves",
+           "SELECT"
+	   " expiration_date"
+	   ",account_details"
+	   ",reserve_pub"
+           ",current_balance_val"
+           ",current_balance_frac"
+           ",current_balance_curr"
+           " FROM reserves"
+           " WHERE expiration_date<=$1"
+	   " AND (current_balance_val != 0 "
+	   "      OR current_balance_frac != 0);",
+           1, NULL);
+
   /* Used in #postgres_get_coin_transactions() to obtain payback transactions
      for a coin */
   PREPARE ("payback_by_coin",
@@ -2069,6 +2088,7 @@ postgres_reserves_in_insert (void *cls,
                              const json_t *sender_account_details,
                              const json_t *transfer_details)
 {
+  struct PostgresClosure *pg = cls;
   PGresult *result;
   int reserve_exists;
   struct TALER_EXCHANGEDB_Reserve reserve;
@@ -2090,8 +2110,26 @@ postgres_reserves_in_insert (void *cls,
     GNUNET_break (0);
     goto rollback;
   }
+  if ( (0 == reserve.balance.value) &&
+       (0 == reserve.balance.fraction) )
+  {
+    /* TODO: reserve balance is empty, we might want to update
+       sender_account_details here.  (So that IF a customer uses the
+       same reserve public key from a different account, we USUALLY
+       switch to the new account (but only if the old reserve was
+       drained).)  This helps make sure that on reserve expiration the
+       funds go back to a valid account in cases where the customer
+       has closed the old bank account and some (buggy?) wallet keeps
+       using the same reserve key with the customer's new account.
+
+       Note that for a non-drained reserve we should not switch,
+       as that opens an attack vector for an adversary who can see
+       the wire transfer subjects (i.e. when using Bitcoin).
+    */
+  }
+  
   expiry = GNUNET_TIME_absolute_add (execution_time,
-                                     TALER_IDLE_RESERVE_EXPIRATION_TIME);
+                                     pg->idle_reserve_expiration_time);
   if (GNUNET_NO == reserve_exists)
   {
     /* New reserve, create balance for the first time; we do this
@@ -2101,6 +2139,7 @@ postgres_reserves_in_insert (void *cls,
        as a foreign key. */
     struct GNUNET_PQ_QueryParam params[] = {
       GNUNET_PQ_query_param_auto_from_type (reserve_pub),
+      TALER_PQ_query_param_json (sender_account_details),
       TALER_PQ_query_param_amount (balance),
       GNUNET_PQ_query_param_absolute_time (&expiry),
       GNUNET_PQ_query_param_end
@@ -2302,6 +2341,7 @@ postgres_insert_withdraw_info (void *cls,
                                struct TALER_EXCHANGEDB_Session *session,
                                const struct TALER_EXCHANGEDB_CollectableBlindcoin *collectable)
 {
+  struct PostgresClosure *pg = cls;
   PGresult *result;
   struct TALER_EXCHANGEDB_Reserve reserve;
   struct GNUNET_HashCode denom_pub_hash;
@@ -2356,7 +2396,7 @@ postgres_insert_withdraw_info (void *cls,
     return GNUNET_NO;
   }
   expiry = GNUNET_TIME_absolute_add (now,
-                                     TALER_IDLE_RESERVE_EXPIRATION_TIME);
+                                     pg->idle_reserve_expiration_time);
   reserve.expiry = GNUNET_TIME_absolute_max (expiry,
                                              reserve.expiry);
   if (GNUNET_OK != reserves_update (cls,
@@ -2618,7 +2658,7 @@ postgres_get_reserve_history (void *cls,
                                                &closing->execution_date),
           TALER_PQ_result_spec_json ("receiver_account",
 				     &closing->receiver_account_details),
-          TALER_PQ_result_spec_json ("transfer_details",
+          GNUNET_PQ_result_spec_auto_from_type ("transfer_details",
 						&closing->transfer_details),
           GNUNET_PQ_result_spec_end
         };
@@ -4934,6 +4974,93 @@ postgres_insert_wire_fee (void *cls,
 }
 
 
+/**
+ * Obtain information about expired reserves and their
+ * remaining balances.
+ *
+ * @param cls closure of the plugin
+ * @param session database connection
+ * @param now timestamp based on which we decide expiration
+ * @param rec function to call on expired reserves
+ * @param rec_cls closure for @a rec
+ * @return #GNUNET_SYSERR on database error
+ *         #GNUNET_NO if there are no expired non-empty reserves
+ *         #GNUNET_OK on success
+ */
+static int
+postgres_get_expired_reserves (void *cls,
+			       struct TALER_EXCHANGEDB_Session *session,
+			       struct GNUNET_TIME_Absolute now,
+			       TALER_EXCHANGEDB_ReserveExpiredCallback rec,
+			       void *rec_cls)
+{
+  struct GNUNET_PQ_QueryParam params[] = {
+    GNUNET_PQ_query_param_absolute_time (&now),
+    GNUNET_PQ_query_param_end
+  };
+  PGresult *result;
+  int nrows;
+
+  result = GNUNET_PQ_exec_prepared (session->conn,
+                                    "get_expired_reserves",
+                                    params);
+  if (PGRES_TUPLES_OK !=
+      PQresultStatus (result))
+  {
+    BREAK_DB_ERR (result, session->conn);
+    PQclear (result);
+    return GNUNET_SYSERR;
+  }
+  nrows = PQntuples (result);
+  if (0 == nrows)
+  {
+    /* no matches found */
+    PQclear (result);
+    return GNUNET_NO;
+  }
+
+  for (int i=0;i<nrows;i++)
+  {
+    struct GNUNET_TIME_Absolute exp_date;
+    json_t *account_details;
+    struct TALER_ReservePublicKeyP reserve_pub;
+    struct TALER_Amount remaining_balance;
+    int ret;
+    struct GNUNET_PQ_ResultSpec rs[] = {
+      GNUNET_PQ_result_spec_absolute_time ("expiration_date",
+					   &exp_date),
+      TALER_PQ_result_spec_json ("account_details",
+				 &account_details),
+      GNUNET_PQ_result_spec_auto_from_type ("reserve_pub",
+					    &reserve_pub),
+      TALER_PQ_result_spec_amount ("current_balance",
+				   &remaining_balance),
+      GNUNET_PQ_result_spec_end
+    };
+  
+    if (GNUNET_OK !=
+	GNUNET_PQ_extract_result (result,
+				  rs,
+				  i))
+    {
+      PQclear (result);
+      GNUNET_break (0);
+      return GNUNET_SYSERR;
+    }
+    ret = rec (rec_cls,
+	       &reserve_pub,
+	       &remaining_balance,
+	       account_details,
+	       exp_date);
+    GNUNET_PQ_cleanup_result (rs);
+    if (GNUNET_OK != ret)
+      break;
+  }
+  PQclear (result);
+  return GNUNET_OK;
+}
+
+
 /**
  * Insert reserve close operation into database.
  *
@@ -4951,23 +5078,25 @@ postgres_insert_wire_fee (void *cls,
 static int
 postgres_insert_reserve_closed (void *cls,
 				struct TALER_EXCHANGEDB_Session *session,
-				struct TALER_ReservePublicKeyP *reserve_pub,
+				const struct TALER_ReservePublicKeyP *reserve_pub,
 				struct GNUNET_TIME_Absolute execution_date,
 				const json_t *receiver_account,
-				const json_t *transfer_details,
+				const struct TALER_WireTransferIdentifierRawP *transfer_details,
 				const struct TALER_Amount *amount_with_fee,
 				const struct TALER_Amount *closing_fee)
 {
+  struct TALER_EXCHANGEDB_Reserve reserve;
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_auto_from_type (reserve_pub),
     GNUNET_PQ_query_param_absolute_time (&execution_date),
-    TALER_PQ_query_param_json (transfer_details),
+    GNUNET_PQ_query_param_auto_from_type (transfer_details),
     TALER_PQ_query_param_json (receiver_account),
     TALER_PQ_query_param_amount (amount_with_fee),
     TALER_PQ_query_param_amount (closing_fee),
     GNUNET_PQ_query_param_end
   };
   PGresult *result;
+  int ret;
   
   result = GNUNET_PQ_exec_prepared (session->conn,
 				    "reserves_close_insert",
@@ -4985,6 +5114,38 @@ postgres_insert_reserve_closed (void *cls,
     return GNUNET_SYSERR;
   }
   PQclear (result);
+
+  /* update reserve balance */
+  reserve.pub = *reserve_pub;
+  if (GNUNET_OK != postgres_reserve_get (cls,
+                                         session,
+                                         &reserve))
+  {
+    /* Should have been checked before we got here... */
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  ret = TALER_amount_subtract (&reserve.balance,
+			       &reserve.balance,
+			       amount_with_fee);
+  if (GNUNET_SYSERR == ret)
+  {
+    /* The reserve history was checked to make sure there is enough of a balance
+       left before we tried this; however, concurrent operations may have changed
+       the situation by now.  We should re-try the transaction.  */
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Closing of reserve `%s' refused due to balance missmatch. Retrying.\n",
+                TALER_B2S (reserve_pub));
+    return GNUNET_NO;
+  }
+  GNUNET_break (GNUNET_NO == ret);
+  if (GNUNET_OK != reserves_update (cls,
+                                    session,
+                                    &reserve))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
   return GNUNET_OK;
 }
 
@@ -6069,7 +6230,7 @@ postgres_select_reserve_closed_above_serial_id (void *cls,
     uint64_t rowid;
     struct TALER_ReservePublicKeyP reserve_pub;
     json_t *receiver_account;
-    json_t *transfer_details;
+    struct TALER_WireTransferIdentifierRawP wtid;
     struct TALER_Amount amount_with_fee;
     struct TALER_Amount closing_fee;
     struct GNUNET_TIME_Absolute execution_date;
@@ -6080,8 +6241,8 @@ postgres_select_reserve_closed_above_serial_id (void *cls,
                                             &reserve_pub),
       GNUNET_PQ_result_spec_absolute_time ("execution_date",
                                            &execution_date),
-      TALER_PQ_result_spec_json ("transfer_details",
+      GNUNET_PQ_result_spec_auto_from_type ("transfer_details",
-				 &transfer_details),
+					    &wtid),
       TALER_PQ_result_spec_json ("receiver_account",
 				 &receiver_account),
       TALER_PQ_result_spec_amount ("amount",
@@ -6107,7 +6268,7 @@ postgres_select_reserve_closed_above_serial_id (void *cls,
 	      &closing_fee,
               &reserve_pub,
 	      receiver_account,
-              transfer_details);
+              &wtid);
     GNUNET_PQ_cleanup_result (rs);
     if (GNUNET_OK != ret)
       break;
@@ -6147,6 +6308,7 @@ postgres_insert_payback_request (void *cls,
                                  const struct GNUNET_HashCode *h_blind_ev,
                                  struct GNUNET_TIME_Absolute timestamp)
 {
+  struct PostgresClosure *pg = cls;
   PGresult *result;
   struct GNUNET_TIME_Absolute expiry;
   struct TALER_EXCHANGEDB_Reserve reserve;
@@ -6215,7 +6377,7 @@ postgres_insert_payback_request (void *cls,
     return GNUNET_SYSERR;
   }
   expiry = GNUNET_TIME_absolute_add (timestamp,
-                                     TALER_IDLE_RESERVE_EXPIRATION_TIME);
+                                     pg->idle_reserve_expiration_time);
   reserve.expiry = GNUNET_TIME_absolute_max (expiry,
                                              reserve.expiry);
   if (GNUNET_OK != reserves_update (cls,
@@ -6464,6 +6626,18 @@ libtaler_plugin_exchangedb_postgres_init (void *cls)
       return NULL;
     }
   }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (cfg,
+					   "exchangedb",
+					   "IDLE_RESERVE_EXPIRATION_TIME",
+					   &pg->idle_reserve_expiration_time))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               "exchangedb",
+                               "IDLE_RESERVE_EXPIRATION_TIME");
+    GNUNET_free (pg);
+    return NULL;
+  }
   plugin = GNUNET_new (struct TALER_EXCHANGEDB_Plugin);
   plugin->cls = pg;
   plugin->get_session = &postgres_get_session;
@@ -6509,6 +6683,7 @@ libtaler_plugin_exchangedb_postgres_init (void *cls)
   plugin->insert_aggregation_tracking = &postgres_insert_aggregation_tracking;
   plugin->insert_wire_fee = &postgres_insert_wire_fee;
   plugin->get_wire_fee = &postgres_get_wire_fee;
+  plugin->get_expired_reserves = &postgres_get_expired_reserves;
   plugin->insert_reserve_closed = &postgres_insert_reserve_closed;
   plugin->wire_prepare_data_insert = &postgres_wire_prepare_data_insert;
   plugin->wire_prepare_data_mark_finished = &postgres_wire_prepare_data_mark_finished;

src/exchangedb/test-exchange-db-postgres.conf

@@ -6,3 +6,14 @@ DB = postgres
 
 #The connection string the plugin has to use for connecting to the database
 DB_CONN_STR = postgres:///talercheck
+
+
+[exchangedb]
+
+# After how long do we close idle reserves?  The exchange
+# and the auditor must agree on this value.  We currently
+# expect it to be globally defined for the whole system,
+# as there is no way for wallets to query this value.  Thus,
+# it is only configurable for testing, and should be treated
+# as constant in production.
+IDLE_RESERVE_EXPIRATION_TIME = 4 weeks

src/exchangedb/test_exchangedb.c

@@ -1612,24 +1612,37 @@ run (void *cls)
                                           &value,
                                           &cbc.h_coin_envelope,
                                           deadline));
+  FAILIF (GNUNET_OK !=
+          plugin->select_payback_above_serial_id (plugin->cls,
+                                                  session,
+                                                  0,
+                                                  &payback_cb,
+                                                  &coin_blind));
+
+  GNUNET_assert (GNUNET_OK ==
+		 TALER_amount_add (&amount_with_fee,
+				   &value,
+				   &value));
   sndr = json_loads ("{ \"account\":\"1\" }", 0, NULL);
-  just = json_loads ("{ \"trans-details\":\"2\" }", 0, NULL);
   GNUNET_assert (GNUNET_OK ==
                  TALER_string_to_amount (CURRENCY ":0.000010",
                                          &fee_closing));
-  GNUNET_assert (GNUNET_OK ==
-                 TALER_string_to_amount (CURRENCY ":1.000010",
-                                         &amount_with_fee));
   FAILIF (GNUNET_OK !=
 	  plugin->insert_reserve_closed (plugin->cls,
 					 session,
 					 &reserve_pub,
 					 GNUNET_TIME_absolute_get (),
-					 sndr /* receiver_account */,
+					 sndr,
-					 just /* transfer_details */,
+					 &wire_out_wtid,
 					 &amount_with_fee,
 					 &fee_closing));
-  json_decref (just);
+  FAILIF (GNUNET_OK !=
+          check_reserve (session,
+                         &reserve_pub,
+			 0,
+			 0,
+                         value.currency));
+  
   json_decref (sndr);  
   result = 7;
   rh = plugin->get_reserve_history (plugin->cls,
@@ -1880,13 +1893,6 @@ run (void *cls)
                                           &cbc.h_coin_envelope,
                                           deadline));
 
-  FAILIF (GNUNET_OK !=
-          plugin->select_payback_above_serial_id (plugin->cls,
-                                                  session,
-                                                  0,
-                                                  &payback_cb,
-                                                  &coin_blind));
-
   auditor_row_cnt = 0;
   FAILIF (GNUNET_OK !=
           plugin->select_refunds_above_serial_id (plugin->cls,

src/include/taler_exchange_service.h

@@ -765,7 +765,7 @@ struct TALER_EXCHANGE_ReserveHistory
       /**
        * Wire transfer details for the outgoing wire transfer.
        */
-      json_t *transfer_details;
+      struct TALER_WireTransferIdentifierRawP wtid;
 
       /**
        * Signature of the coin of type

src/include/taler_exchangedb_lib.h

@@ -23,6 +23,7 @@
 #ifndef TALER_EXCHANGEDB_LIB_H
 #define TALER_EXCHANGEDB_LIB_H
 
+
 #include "taler_signatures.h"
 
 

src/include/taler_exchangedb_plugin.h

@@ -100,7 +100,7 @@ struct TALER_EXCHANGEDB_ClosingTransfer
    * Detailed wire transfer information that uniquely identifies the
    * wire transfer.
    */
-  json_t *transfer_details;
+  struct TALER_WireTransferIdentifierRawP transfer_details;
   
 };
 
@@ -991,7 +991,25 @@ typedef int
 					  const struct TALER_Amount *closing_fee,
 					  const struct TALER_ReservePublicKeyP *reserve_pub,
 					  const json_t *receiver_account,
-					  const json_t *transfer_details);
+					  const struct TALER_WireTransferIdentifierRawP *transfer_details);
+
+
+/**
+ * Function called with details about expired reserves.
+ *
+ * @param cls closure
+ * @param reserve_pub public key of the reserve
+ * @param left amount left in the reserve
+ * @param account_details information about the reserve's bank account
+ * @param expiration_date when did the reserve expire
+ * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop
+ */
+typedef int
+(*TALER_EXCHANGEDB_ReserveExpiredCallback)(void *cls,
+					   const struct TALER_ReservePublicKeyP *reserve_pub,
+					   const struct TALER_Amount *left,
+					   const json_t *account_details,
+					   struct GNUNET_TIME_Absolute expiration_date);
 
 
 /**
@@ -1783,6 +1801,27 @@ struct TALER_EXCHANGEDB_Plugin
                    struct TALER_MasterSignatureP *master_sig);
 
 
+  /**
+   * Obtain information about expired reserves and their
+   * remaining balances.
+   *
+   * @param cls closure of the plugin
+   * @param session database connection
+   * @param now timestamp based on which we decide expiration
+   * @param rec function to call on expired reserves
+   * @param rec_cls closure for @a rec
+   * @return #GNUNET_SYSERR on database error
+   *         #GNUNET_NO if there are no expired non-empty reserves
+   *         #GNUNET_OK on success
+   */
+  int
+  (*get_expired_reserves)(void *cls,
+			  struct TALER_EXCHANGEDB_Session *session,
+			  struct GNUNET_TIME_Absolute now,
+			  TALER_EXCHANGEDB_ReserveExpiredCallback rec,
+			  void *rec_cls);
+
+  
   /**
    * Insert reserve close operation into database.
    *
@@ -1800,10 +1839,10 @@ struct TALER_EXCHANGEDB_Plugin
   int
   (*insert_reserve_closed)(void *cls,
 			   struct TALER_EXCHANGEDB_Session *session,
-			   struct TALER_ReservePublicKeyP *reserve_pub,
+			   const struct TALER_ReservePublicKeyP *reserve_pub,
 			   struct GNUNET_TIME_Absolute execution_date,
 			   const json_t *receiver_account,
-			   const json_t *transfer_details,
+			   const struct TALER_WireTransferIdentifierRawP *transfer_details,
 			   const struct TALER_Amount *amount_with_fee,
 			   const struct TALER_Amount *closing_fee);
 

src/include/taler_signatures.h

@@ -46,11 +46,6 @@
  */
 #define TALER_CNC_KAPPA 3
 
-/**
- * After what time do idle reserves "expire"?  We might want to make
- * this a configuration option (eventually).
- */
-#define TALER_IDLE_RESERVE_EXPIRATION_TIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_YEARS, 5)
 
 /*********************************************/
 /* Exchange offline signatures (with master key) */
@@ -1269,9 +1264,9 @@ struct TALER_ReserveCloseConfirmationPS
   struct GNUNET_HashCode h_wire;
 
   /**
-   * Hash of the transfer details.
+   * Wire transfer subject.
    */
-  struct GNUNET_HashCode h_transfer;
+  struct TALER_WireTransferIdentifierRawP wtid;
 };