WIP: policy_*_fulfiment added and API changes

- policy_fulfilment table defined, handlers added
- policy_details_fulfilment table defined, handlers added
- TALER_extensions_serial_from_policy_details implemened

src/auditor/taler-auditor-sync.c

@@ -115,6 +115,7 @@ static struct Table tables[] = {
   { .rt = TALER_EXCHANGEDB_RT_EXTENSIONS},
   { .rt = TALER_EXCHANGEDB_RT_POLICY_DETAILS },
   { .rt = TALER_EXCHANGEDB_RT_POLICY_FULFILMENTS },
+  { .rt = TALER_EXCHANGEDB_RT_POLICY_DETAILS_FULFILMENTS },
   { .rt = TALER_EXCHANGEDB_RT_PURSE_REQUESTS},
   { .rt = TALER_EXCHANGEDB_RT_PURSE_REFUNDS},
   { .rt = TALER_EXCHANGEDB_RT_PURSE_MERGES},

src/auditor/taler-helper-auditor-coins.c

@@ -1620,14 +1620,14 @@ deposit_cb (void *cls,
        want to do in parallel in the background to improve
        auditor performance! */
     if (GNUNET_OK !=
-        TALER_wallet_deposit_verify (&deposit->amount_with_fee,
+        TALER_wallet_deposit_verify (
+          &deposit->amount_with_fee,
           &issue->fees.deposit,
           &h_wire,
           &deposit->h_contract_terms,
           &deposit->coin.h_age_commitment,
-                                     deposit->no_policy_details ? NULL :
+          deposit->has_policy_details ?
-                                     &deposit->h_policy,
+          &deposit->h_policy :NULL, &h_denom_pub,
-                                     &h_denom_pub,
           deposit->timestamp,
           &deposit->merchant_pub,
           deposit->refund_deadline,

src/exchange/taler-exchange-httpd_batch-deposit.c

@@ -91,7 +91,7 @@ struct BatchDepositContext
    * deposit operation, possibly NULL!
    */
   json_t *policy_details;
-  bool no_policy_details;
+  bool has_policy_details;
 
   /**
    * Hash over @e policy_details, might be all zero;
@@ -174,7 +174,7 @@ again:
            &TEH_keys_exchange_sign_,
            &bdc->h_contract_terms,
            &bdc->h_wire,
-           bdc->no_policy_details ? NULL : &bdc->h_policy,
+           bdc->has_policy_details ? &bdc->h_policy : NULL,
            bdc->exchange_timestamp,
            bdc->wire_deadline,
            bdc->refund_deadline,
@@ -474,7 +474,8 @@ parse_coin (struct MHD_Connection *connection,
                                    &dc->h_wire,
                                    &dc->h_contract_terms,
                                    &deposit->coin.h_age_commitment,
-                                   dc->no_policy_details ? NULL : &dc->h_policy,
+                                   dc->has_policy_details ? &dc->h_policy :
+                                   NULL,
                                    &deposit->coin.denom_pub_hash,
                                    dc->timestamp,
                                    &dc->merchant_pub,
@@ -517,6 +518,7 @@ TEH_handler_batch_deposit (struct TEH_RequestContext *rc,
   struct BatchDepositContext dc;
   json_t *coins;
   bool no_refund_deadline = true;
+  bool no_policy_details = true;
   struct GNUNET_JSON_Specification spec[] = {
     GNUNET_JSON_spec_string ("merchant_payto_uri",
                              &dc.payto_uri),
@@ -531,7 +533,7 @@ TEH_handler_batch_deposit (struct TEH_RequestContext *rc,
     GNUNET_JSON_spec_mark_optional (
       GNUNET_JSON_spec_json ("policy",
                              &dc.policy_details),
-      &dc.no_policy_details),
+      &no_policy_details),
     GNUNET_JSON_spec_timestamp ("timestamp",
                                 &dc.timestamp),
     GNUNET_JSON_spec_mark_optional (
@@ -562,6 +564,8 @@ TEH_handler_batch_deposit (struct TEH_RequestContext *rc,
     return MHD_YES;   /* failure */
   }
 
+  dc.has_policy_details = ! no_policy_details;
+
   /* validate merchant's wire details (as far as we can) */
   {
     char *emsg;
@@ -606,7 +610,7 @@ TEH_handler_batch_deposit (struct TEH_RequestContext *rc,
   TALER_merchant_wire_signature_hash (dc.payto_uri,
                                       &dc.wire_salt,
                                       &dc.h_wire);
-  if (! dc.no_policy_details)
+  if (dc.has_policy_details)
   {
     TALER_deposit_policy_hash (dc.policy_details,
                                &dc.h_policy);

src/exchange/taler-exchange-httpd_deposit.c

@@ -21,6 +21,7 @@
  * @author Florian Dold
  * @author Benedikt Mueller
  * @author Christian Grothoff
+ * @author Özgür Kesim
  */
 #include "platform.h"
 #include <gnunet/gnunet_util_lib.h>
@@ -218,6 +219,7 @@ TEH_handler_deposit (struct MHD_Connection *connection,
   struct TALER_EXCHANGEDB_Deposit deposit;
   const char *payto_uri;
   struct TALER_ExtensionPolicyHashP *ph_policy = NULL;
+  bool no_policy_details;
   struct GNUNET_JSON_Specification spec[] = {
     GNUNET_JSON_spec_string ("merchant_payto_uri",
                              &payto_uri),
@@ -254,7 +256,7 @@ TEH_handler_deposit (struct MHD_Connection *connection,
     GNUNET_JSON_spec_mark_optional (
       GNUNET_JSON_spec_json ("policy",
                              &deposit.policy_details),
-      &deposit.no_policy_details),
+      &no_policy_details),
     GNUNET_JSON_spec_end ()
   };
   struct TALER_MerchantWireHashP h_wire;
@@ -328,31 +330,15 @@ TEH_handler_deposit (struct MHD_Connection *connection,
   dc.deposit = &deposit;
 
   /* Check policy */
-  if (! deposit.no_policy_details)
+  deposit.has_policy_details = ! no_policy_details;
+  if (! deposit.has_policy_details)
   {
-    enum GNUNET_GenericReturnValue ret;
-    const struct TALER_Extension *ext;
     const char *error_hint = NULL;
-    GNUNET_assert (ext->parse_policy_details);
+    if (GNUNET_OK !=
-
+        TALER_extensions_serial_from_policy_details (deposit.policy_details,
-    do {
-      ret = TALER_extensions_from_policy_details (deposit.policy_details,
-                                                  &ext,
-                                                  &error_hint);
-
-      if (GNUNET_OK != ret)
-        break;
-
-      deposit.policy_deadline = GNUNET_TIME_UNIT_FOREVER_TS;
-
-      ret = ext->parse_policy_details (deposit.policy_details,
                                                      &deposit.policy_serial_id,
                                                      &deposit.policy_deadline,
-                                       &error_hint);
+                                                     &error_hint))
-
-    } while(0);
-
-    if (GNUNET_OK != ret)
       return TALER_MHD_reply_with_error (connection,
                                          MHD_HTTP_BAD_REQUEST,
                                          TALER_EC_EXCHANGE_DEPOSITS_POLICY_NOT_ACCEPTED,
@@ -424,7 +410,7 @@ TEH_handler_deposit (struct MHD_Connection *connection,
                                          NULL);
     }
 
-    if (! deposit.no_policy_details)
+    if (deposit.has_policy_details)
     {
       TALER_deposit_policy_hash (deposit.policy_details,
                                  &deposit.h_policy);

src/exchangedb/exchange-0001-part.sql

@@ -539,10 +539,10 @@ SELECT add_constraints_to_refresh_transfer_keys_partition('default');
 -- ------------------------------ policy_fulfilments -------------------------------------
 
 CREATE TABLE IF NOT EXISTS policy_fulfilments
-  (policy_fulfilments_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY
+  (fulfilment_id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY
   ,fulfilment_timestamp INT8 NOT NULL
   ,fulfilment_proof VARCHAR)
-  PARTITION BY HASH (policy_fulfilments_serial_id);
+  PARTITION BY HASH (fulfilment_id);
 COMMENT ON TABLE policy_fulfilments
   IS 'Proofs of fulfilment of policies that were set in deposits'; 
 COMMENT ON COLUMN policy_fulfilments.fulfilment_timestamp
@@ -554,14 +554,14 @@ CREATE TABLE IF NOT EXISTS policy_fulfilments_default
   PARTITION OF policy_fulfilments
   FOR VALUES WITH (MODULUS 1, REMAINDER 0);
 
-
 -- ------------------------------ policy_details ----------------------------------------
 
 CREATE TABLE IF NOT EXISTS policy_details
   (policy_details_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY 
   ,serial_id BYTEA PRIMARY KEY CHECK(LENGTH(serial_id)=64)
   ,policy_options VARCHAR
-  ,fulfilment_serial_id BIGINT REFERENCES policy_fulfilments(policy_fulfilments_serial_id) ON DELETE CASCADE)
+  ,deadline INT8 NOT NULL
+  ,fulfilment_state INT4 NOT NULL CHECK(fulfilment_state between 0 and 3))
   PARTITION BY HASH (serial_id);
 COMMENT ON TABLE policy_details
   IS 'Policies that were provided with deposits via policy extensions.';
@@ -569,13 +569,34 @@ COMMENT ON COLUMN policy_details.serial_id
   IS 'ID (GNUNET_HashCode) that identifies a policy.  Will be calculated by the policy extension based on the content';
 COMMENT ON COLUMN policy_details.policy_options
   IS 'JSON object with options set that the exchange needs to consider when executing a deposit. Supported details depend on the policy extensions supported by the exchange.';
-COMMENT ON COLUMN policy_details.fulfilment_serial_id
+COMMENT ON COLUMN policy_details.deadline
-  IS 'If not NULL, refers to the proof of fulfilment of this policy';
+  IS 'Deadline until the policy must be marked as fulfilled or unfulfilled (maybe "forever")';
+COMMENT ON COLUMN policy_details.fulfilment_state
+  IS 'State of the fulfilment: 0 (PENDING), 1 (FULFILLED), 2 (NOT FULFILLED), 3 (TIMED OUT)';
 
 CREATE TABLE IF NOT EXISTS policy_details_default
   PARTITION OF policy_details
   FOR VALUES WITH (MODULUS 1, REMAINDER 0);
 
+-- ------------------------------ policy_details_fulfilments -----------------------------
+
+CREATE TABLE IF NOT EXISTS policy_details_fulfilments
+  (fulfilment_id BIGINT NOT NULL REFERENCES policy_fulfilments(fulfilment_id) ON DELETE CASCADE
+  ,serial_id BYTEA NOT NULL UNIQUE REFERENCES policy_details(serial_id) ON DELETE CASCADE)
+  PARTITION BY HASH (serial_id); -- FIXME: choose other thing to hash here?
+-- FIXME: define a primary key here?
+COMMENT ON TABLE policy_details_fulfilments
+  IS 'Links policy_details.serial_id''s with policy_fulfilments.id''s. The same proof of fulfilment can be associated with multiple serial-id''s';
+COMMENT ON COLUMN policy_details_fulfilments.fulfilment_id
+  IS 'ID of the proof of fulfilment';
+COMMENT ON COLUMN policy_details_fulfilments.serial_id
+  IS 'Serial-ID of the corresponding policy_detail';
+
+CREATE TABLE IF NOT EXISTS policy_details_fulfilments_default
+  PARTITION OF policy_details_fulfilments
+  FOR VALUES WITH (MODULUS 1, REMAINDER 0);
+
+
 -- ------------------------------ deposits ----------------------------------------
 
 SELECT create_table_deposits();

src/exchangedb/lrbt_callbacks.c

@@ -1442,7 +1442,6 @@ lrbt_cb_table_policy_details (void *cls,
   for (unsigned int i = 0; i<num_results; i++)
   {
     bool no_config = false;
-    bool no_fulfilment = false;
     struct GNUNET_PQ_ResultSpec rs[] = {
       GNUNET_PQ_result_spec_uint64 ("policy_details_serial_id",
                                     &td.serial),
@@ -1454,11 +1453,12 @@ lrbt_cb_table_policy_details (void *cls,
                                       &td.details.policy_details.
                                       policy_options),
         &no_config),
-      GNUNET_PQ_result_spec_allow_null (
+      GNUNET_PQ_result_spec_timestamp ("deadline",
-        GNUNET_PQ_result_spec_uint64 ("fulfilment_serial_id",
                                        &td.details.policy_details.
-                                      fulfilment_serial_id),
+                                       deadline),
-        &no_fulfilment),
+      GNUNET_PQ_result_spec_uint64 ("fulfilment_state",
+                                    &td.details.policy_details.
+                                    fulfilment_state),
       GNUNET_PQ_result_spec_end
     };
 
@@ -1500,7 +1500,7 @@ lrbt_cb_table_policy_fulfilments (void *cls,
     bool no_config = false;
     bool no_timestamp = false;
     struct GNUNET_PQ_ResultSpec rs[] = {
-      GNUNET_PQ_result_spec_uint64 ("policy_fulfilment_serial_id",
+      GNUNET_PQ_result_spec_uint64 ("fulfilment_id",
                                     &td.serial),
       GNUNET_PQ_result_spec_allow_null (
         GNUNET_PQ_result_spec_string ("fulfilment_proof",
@@ -1531,6 +1531,52 @@ lrbt_cb_table_policy_fulfilments (void *cls,
 }
 
 
+/**
+ * Function called with policy_details_fulfilments table entries.
+ *
+ * @param cls closure
+ * @param result the postgres result
+ * @param num_results the number of results in @a result
+ */
+static void
+lrbt_cb_table_policy_details_fulfilments (void *cls,
+                                          PGresult *result,
+                                          unsigned int num_results)
+{
+  struct LookupRecordsByTableContext *ctx = cls;
+  struct TALER_EXCHANGEDB_TableData td = {
+    .table = TALER_EXCHANGEDB_RT_POLICY_DETAILS_FULFILMENTS
+  };
+
+  for (unsigned int i = 0; i<num_results; i++)
+  {
+    struct GNUNET_PQ_ResultSpec rs[] = {
+      GNUNET_PQ_result_spec_uint64 ("fulfilment_id",
+                                    &td.details.policy_details_fulfilments.
+                                    fulfilment_id),
+      GNUNET_PQ_result_spec_auto_from_type ("serial_id",
+                                            &td.details.
+                                            policy_details_fulfilments.
+                                            serial_id),
+      GNUNET_PQ_result_spec_end
+    };
+
+    if (GNUNET_OK !=
+        GNUNET_PQ_extract_result (result,
+                                  rs,
+                                  i))
+    {
+      GNUNET_break (0);
+      ctx->error = true;
+      return;
+    }
+    ctx->cb (ctx->cb_cls,
+             &td);
+    GNUNET_PQ_cleanup_result (rs);
+  }
+}
+
+
 /**
  * Function called with purse_requests table entries.
  *

src/exchangedb/plugin_exchangedb_postgres.c

@@ -3932,19 +3932,28 @@ prepare_statements (struct PostgresClosure *pg)
       "(policy_details_serial_id"
       ",serial_id"
       ",policy_options"
-      ",fulfilment_serial_id"
+      ",deadline"
+      ",fulfilment_state"
       ") VALUES "
-      "($1, $2, $3, $4);",
+      "($1, $2, $3, $4, $5);",
-      4),
+      5),
     GNUNET_PQ_make_prepare (
       "insert_into_table_policy_fulfilments",
       "INSERT INTO policy_fulfilments"
-      "(policy_fulfilments_serial_id"
+      "(fulfilment_id"
       ",fulfilment_timestamp"
       ",fulfilment_proof"
       ") VALUES "
       "($1, $2, $3);",
       3),
+    GNUNET_PQ_make_prepare (
+      "insert_into_table_policy_details_fulfilments",
+      "INSERT INTO policy_details_fulfilments"
+      "(fulfilment_id"
+      ",serial_id"
+      ") VALUES "
+      "($1, $2);",
+      2),
     GNUNET_PQ_make_prepare (
       "insert_into_table_purse_requests",
       "INSERT INTO purse_requests"
@@ -6273,16 +6282,16 @@ postgres_do_deposit (
     GNUNET_PQ_query_param_auto_from_type (&deposit->coin.coin_pub),
     GNUNET_PQ_query_param_auto_from_type (&deposit->csig),
     GNUNET_PQ_query_param_uint64 (&deposit_shard),
-    GNUNET_PQ_query_param_bool (! deposit->no_policy_details),
+    GNUNET_PQ_query_param_bool (deposit->has_policy_details),
-    (deposit->no_policy_details)
+    (deposit->has_policy_details)
-    ? GNUNET_PQ_query_param_null ()
+    ? TALER_PQ_query_param_json (deposit->policy_details)
-    : TALER_PQ_query_param_json (deposit->policy_details),
+    : GNUNET_PQ_query_param_null (),
-    (deposit->no_policy_details)
+    (deposit->has_policy_details)
-    ? GNUNET_PQ_query_param_null ()
+    ? GNUNET_PQ_query_param_auto_from_type (&deposit->policy_serial_id)
-    : GNUNET_PQ_query_param_auto_from_type (&deposit->policy_serial_id),
+    : GNUNET_PQ_query_param_null (),
-    (deposit->no_policy_details)
+    (deposit->has_policy_details)
-    ? GNUNET_PQ_query_param_null ()
+    ? GNUNET_PQ_query_param_timestamp (&deposit->policy_deadline)
-    : GNUNET_PQ_query_param_timestamp (&deposit->policy_deadline),
+    : GNUNET_PQ_query_param_null (),
     GNUNET_PQ_query_param_end
   };
   struct GNUNET_PQ_ResultSpec rs[] = {

src/exchangedb/procedures.sql

@@ -512,6 +512,8 @@ CREATE OR REPLACE FUNCTION exchange_do_deposit(
   IN in_shard INT8,
   IN in_policy_blocked BOOLEAN,
   IN in_policy_details VARCHAR,
+  IN in_policy_serial_id BYTEA,
+  IN in_policy_deadline INT8,
   OUT out_exchange_timestamp INT8,
   OUT out_balance_ok BOOLEAN,
   OUT out_conflict BOOLEAN)
@@ -530,9 +532,15 @@ BEGIN
 IF NOT NULL in_policy_details
 THEN
   INSERT INTO exchange.policy_details
-  (policy_options)
+  (serial_id
+  ,policy_options
+  ,deadline
+  ,fulfilment_state)
   VALUES
-    (in_policy_details)
+    (in_policy_serial_id
+    ,in_policy_details
+    ,in_policy_deadline
+    ,'pending')
   RETURNING policy_details_serial_id INTO xdi;
 ELSE
   xdi=NULL;

src/exchangedb/test_exchangedb.c

@@ -1500,7 +1500,6 @@ run (void *cls)
                                 &deposit,
                                 known_coin_id,
                                 &h_payto,
-                                false,
                                 &deposit_timestamp,
                                 &balance_ok,
                                 &in_conflict));

src/extensions/extensions.c

@@ -354,15 +354,16 @@ TALER_extensions_load_manifests (
 
 
 enum GNUNET_GenericReturnValue
-TALER_extensions_from_policy_details (
+TALER_extensions_serial_from_policy_details (
   const json_t *policy_details,
-  const struct TALER_Extension **extension,
+  struct GNUNET_HashCode *serial,
+  struct GNUNET_TIME_Timestamp *deadline,
   const char **error_hint)
 {
+  const struct TALER_Extension *extension;
   const json_t *jtype;
   const char *type;
 
-  *extension = NULL;
   *error_hint = NULL;
 
   if ((NULL == policy_details) ||
@@ -386,19 +387,23 @@ TALER_extensions_from_policy_details (
     return GNUNET_SYSERR;
   }
 
-  *extension = TALER_extensions_get_by_name (type);
+  extension = TALER_extensions_get_by_name (type);
-  if ((NULL == *extension) ||
+  if ((NULL == extension) ||
-      (NULL == (*extension)->parse_policy_details))
+      (NULL == extension->parse_policy_details))
   {
     GNUNET_break (0);
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 "Unsupported extension policy '%s' requested\n",
                 type);
-    *extension = NULL;
     return GNUNET_NO;
   }
 
-  return GNUNET_OK;
+  *deadline = GNUNET_TIME_UNIT_FOREVER_TS;
+  return extension->parse_policy_details (policy_details,
+                                          serial,
+                                          deadline,
+                                          error_hint);
+
 }
 
 

src/include/taler_exchangedb_plugin.h

@@ -223,6 +223,7 @@ enum TALER_EXCHANGEDB_ReplicatedTable
   TALER_EXCHANGEDB_RT_EXTENSIONS,
   TALER_EXCHANGEDB_RT_POLICY_DETAILS,
   TALER_EXCHANGEDB_RT_POLICY_FULFILMENTS,
+  TALER_EXCHANGEDB_RT_POLICY_DETAILS_FULFILMENTS,
   TALER_EXCHANGEDB_RT_PURSE_REQUESTS,
   TALER_EXCHANGEDB_RT_PURSE_REFUNDS,
   TALER_EXCHANGEDB_RT_PURSE_MERGES,
@@ -530,9 +531,15 @@ struct TALER_EXCHANGEDB_TableData
       char *policy_options;
       struct GNUNET_HashCode serial_id;
       struct GNUNET_TIME_Timestamp deadline;
-      uint64_t fulfilment_serial_id;
+      uint64_t fulfilment_state;
     } policy_details;
 
+    struct
+    {
+      struct GNUNET_HashCode serial_id;
+      uint64_t fulfilment_id;
+    } policy_details_fulfilments;
+
     struct
     {
       char *fulfilment_proof;
@@ -1442,23 +1449,25 @@ struct TALER_EXCHANGEDB_Deposit
    * deposit operation, possibly NULL!
    */
   json_t *policy_details;
-  bool no_policy_details;
+  bool has_policy_details;
 
   /**
-   * If policy_details are present, the corresponding policy extension calculates
+   * If policy_details are present, the corresponding policy extension
-   * a serial id under which the policy_details shall be stored in the policy_details table.
+   * calculates a serial id under which the policy_details shall be stored in
+   * the policy_details table.
    */
   struct GNUNET_HashCode policy_serial_id;
 
   /**
-   * If policy_details are present, the corresponding policy extension can
+   * If policy_details are present, the corresponding policy extension can set
-   * set a deadline for this policy.  Can be "forever".
+   * a deadline for this policy.  Can be "forever".
    */
   struct GNUNET_TIME_Timestamp policy_deadline;
 
 
   /**
-   * Hash over the @e policy_details.  Only filled if no_policy_details is false.
+   * Hash over the @e policy_details.  Only filled if has_policy_details is
+   * true.
    */
   struct TALER_ExtensionPolicyHashP h_policy;
 

src/include/taler_extensions.h

@@ -390,16 +390,18 @@ TALER_extensions_get_age_restriction_mask ();
  * @brief Finds the extension for a given policy
  *
  * @param[in]  policy_details JSON of the policy detail from a deposit request
- * @param[out] extension On GNUNET_OK, the exentions handling the given policy
+ * @param[out] serial On GNUNET_OK, the hash code that should be used to save the policy_details in the policy_details table
+ * @param[out] deadline On GNUNET_OK, the deadline that should be saved in the policy_details table
  * @param[out] error_hint On GNUNET_SYSERR, will contain a hint for the reason why it failed
  * @return GNUNET_OK on success, with *extension set to the correct extension.
  * GNUNET_NO, when no extension was found. GNUNET_SYSERR when the JSON was
  * invalid, with *error_hint maybe non-NULL.
  */
 enum GNUNET_GenericReturnValue
-TALER_extensions_from_policy_details (
+TALER_extensions_serial_from_policy_details (
   const json_t *policy_details,
-  const struct TALER_Extension **extension,
+  struct GNUNET_HashCode *serial,
+  struct GNUNET_TIME_Timestamp *deadline,
   const char **error_hint);
 
 /*