add selection by shard, ignore KYC in revocation test aggregation

src/auditor/test-revocation.sh

@@ -74,8 +74,9 @@ function pre_audit () {
 
     if test ${1:-no} = "aggregator"
     then
+        export CONF
         echo -n "Running exchange aggregator ..."
-        taler-exchange-aggregator -L INFO -t -c $CONF 2> aggregator.log || exit_fail "FAIL"
+        taler-exchange-aggregator -L INFO -t -c $CONF -y 2> aggregator.log || exit_fail "FAIL"
         echo " DONE"
         echo -n "Running exchange closer ..."
         taler-exchange-closer -L INFO -t -c $CONF 2> closer.log || exit_fail "FAIL"

src/exchangedb/exchange-0001.sql

@@ -113,6 +113,7 @@ CREATE INDEX IF NOT EXISTS wire_targets_serial_id_index
   ON wire_targets
   (wire_target_serial_id);
 
+
 CREATE TABLE IF NOT EXISTS reserves
   (reserve_uuid BIGINT GENERATED BY DEFAULT AS IDENTITY
   ,reserve_pub BYTEA PRIMARY KEY CHECK(LENGTH(reserve_pub)=32)
@@ -623,7 +624,7 @@ CREATE TABLE IF NOT EXISTS deposits
   ,extension_details_serial_id INT8 REFERENCES extension_details (extension_details_serial_id) ON DELETE CASCADE
   ,UNIQUE (shard, coin_pub, merchant_pub, h_contract_terms)
   )
-  PARTITION BY HASH (shard);
+  PARTITION BY HASH (shard); -- FIXME: why not BY RANGE? RANGE would seem better for 'deposits_get_ready'!
 
 CREATE TABLE IF NOT EXISTS deposits_default
   PARTITION OF deposits
@@ -680,11 +681,10 @@ CREATE INDEX IF NOT EXISTS deposits_for_get_ready_index
   );
 COMMENT ON INDEX deposits_for_get_ready_index
   IS 'for deposits_get_ready';
--- FIXME: check if we can ALWAYS include the shard in the WHERE clauses,
--- thereby resulting in a much better use of the index: we could do (shard,merchant_pub, ...)!
 CREATE INDEX IF NOT EXISTS deposits_for_iterate_matching_index
   ON deposits
-  (merchant_pub
+  (shard
+  ,merchant_pub
   ,wire_target_h_payto
   ,done
   ,extension_blocked
@@ -786,7 +786,7 @@ CREATE INDEX IF NOT EXISTS wire_out_by_wire_target_h_payto_index
 
 CREATE TABLE IF NOT EXISTS aggregation_tracking
   (aggregation_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY -- UNIQUE
-  ,deposit_serial_id INT8 PRIMARY KEY -- REFERENCES deposits (deposit_serial_id) ON DELETE CASCADE
+  ,deposit_serial_id INT8 PRIMARY KEY -- REFERENCES deposits (deposit_serial_id) ON DELETE CASCADE -- FIXME: change to coin_pub + deposit_serial_id for more efficient deposit -- or something else??? 
   ,wtid_raw BYTEA NOT NULL CONSTRAINT wire_out_ref REFERENCES wire_out(wtid_raw) ON DELETE CASCADE DEFERRABLE
   )
   PARTITION BY HASH (deposit_serial_id);

src/exchangedb/plugin_exchangedb_postgres.c

@@ -1193,8 +1193,8 @@ prepare_statements (struct PostgresClosure *pg)
       " FROM deposits"
       "    JOIN known_coins kc USING (coin_pub)"
       "    JOIN denominations denom USING (denominations_serial)"
-      " WHERE"
+      " WHERE shard=$4"
-      "      merchant_pub=$1"
+      "  AND merchant_pub=$1"
       "  AND wire_target_h_payto=$2"
       "  AND done=FALSE"
       "  AND extension_blocked=FALSE"
@@ -1203,7 +1203,7 @@ prepare_statements (struct PostgresClosure *pg)
       " LIMIT "
       TALER_QUOTE (
         TALER_EXCHANGEDB_MATCHING_DEPOSITS_LIMIT) ";",
-      3),
+      4),
     /* Used in #postgres_mark_deposit_tiny() */
     GNUNET_PQ_make_prepare (
       "mark_deposit_tiny",
@@ -1303,6 +1303,7 @@ prepare_statements (struct PostgresClosure *pg)
       " WHERE wtid_raw=$1;",
       1),
     /* Used in #postgres_lookup_transfer_by_deposit */
+    // FIXME: select by shard?
     GNUNET_PQ_make_prepare (
       "lookup_deposit_wtid",
       "SELECT"
@@ -1483,6 +1484,10 @@ prepare_statements (struct PostgresClosure *pg)
       " LIMIT $2;",
       2),
     /* Used in #postgres_select_deposits_missing_wire */
+    // FIXME: used by the auditor; can probably be done
+    // smarter by checking if 'done' or 'tiny' or 'blocked'
+    // are set correctly when going over deposits, instead
+    // of JOINing with refunds.
     GNUNET_PQ_make_prepare (
       "deposits_get_overdue",
       "SELECT"
@@ -5870,10 +5875,12 @@ postgres_iterate_matching_deposits (
 {
   struct PostgresClosure *pg = cls;
   struct GNUNET_TIME_Absolute now = GNUNET_TIME_absolute_get ();
+  uint64_t shard = compute_shard (merchant_pub);
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_auto_from_type (merchant_pub),
     GNUNET_PQ_query_param_auto_from_type (h_payto),
     GNUNET_PQ_query_param_absolute_time (&now),
+    GNUNET_PQ_query_param_uint64 (&shard),
     GNUNET_PQ_query_param_end
   };
   struct MatchingDepositContext mdc = {