starting point for NG exchange DB schema

2022-11-24 12:23:55 +01:00 · 2022-11-24 12:23:55 +01:00 · c2bb6551cf
commit c2bb6551cf
parent 0429b0cede
9 changed files with 325 additions and 5 deletions
--- a/contrib/gana
+++ b/contrib/gana
@ -1 +1 @@
-Subproject commit a4a6b9ba4b2634c56194d53e36344686d7052cef
+Subproject commit b971fd44ebf51a9c403dbbff144b43e73aa02c6c
--- a/src/exchangedb/.gitignore
+++ b/src/exchangedb/.gitignore
@ -10,3 +10,5 @@ shard-0000.sql
 shard-0001.sql
 drop0001.sql
 shard-drop0001.sqltest-exchangedb-by-j-postgres
+test-exchangedb-by-j-postgres
+perf-exchangedb-reserves-in-insert-postgres
--- a/src/exchangedb/Makefile.am
+++ b/src/exchangedb/Makefile.am
@ -19,7 +19,12 @@ sqlinputs = \
  exchange-0001.sql.in \
  exchange-0001-part.sql \
  shard-0001-part.sql \
-  shard-0001.sql.in
+  shard-0001.sql.in \
+  common-0002.sql \
+  exchange-0002.sql.in \
+  exchange-0002-part.sql \
+  shard-0002-part.sql \
+  shard-0002.sql.in

 sql_DATA = \
  benchmark-0001.sql \
@ -37,18 +42,30 @@ BUILT_SOURCES = \

 CLEANFILES = \
  shard-0001.sql \
-  exchange-0001.sql
+  exchange-0001.sql \
+  shard-0002.sql \
+  exchange-0002.sql 

 exchange-0001.sql: common-0001.sql exchange-0001-part.sql exchange-0001.sql.in
 	chmod +w $@ || true
 	gcc -E -P -undef - < exchange-0001.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
 	chmod ugo-w $@

+exchange-0002.sql: common-0002.sql exchange-0002-part.sql exchange-0002.sql.in
+	chmod +w $@ || true
+	gcc -E -P -undef - < exchange-0002.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
+	chmod ugo-w $@
+
 shard-0001.sql: common-0001.sql shard-0001-part.sql exchange-0001.sql.in
 	chmod +w $@ || true
 	gcc -E -P -undef - < shard-0001.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
 	chmod ugo-w $@

+shard-0002.sql: common-0002.sql shard-0002-part.sql exchange-0002.sql.in
+	chmod +w $@ || true
+	gcc -E -P -undef - < shard-0002.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
+	chmod ugo-w $@
+
 EXTRA_DIST = \
  exchangedb.conf \
  exchangedb-postgres.conf \
--- a/src/exchangedb/common-0002.sql
+++ b/src/exchangedb/common-0002.sql
@ -0,0 +1,174 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2014--2022 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+--
+
+-------------------- Tables ----------------------------
+
+CREATE OR REPLACE FUNCTION create_table_wire_targets(
+  IN shard_suffix VARCHAR DEFAULT NULL
+)
+RETURNS VOID
+LANGUAGE plpgsql
+AS $$
+BEGIN
+
+  PERFORM create_partitioned_table(
+    'CREATE TABLE IF NOT EXISTS %I'
+      '(wire_target_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY' -- UNIQUE'
+      ',wire_target_h_payto BYTEA PRIMARY KEY CHECK (LENGTH(wire_target_h_payto)=32)'
+      ',payto_uri VARCHAR NOT NULL'
+    ') %s ;'
+    ,'wire_targets'
+    ,'PARTITION BY HASH (wire_target_h_payto)'
+    ,shard_suffix
+  );
+
+END
+$$;
+
+-- We need a separate function for this, as we call create_table only once but need to add
+-- those constraints to each partition which gets created
+CREATE OR REPLACE FUNCTION add_constraints_to_wire_targets_partition(
+  IN partition_suffix VARCHAR
+)
+RETURNS void
+LANGUAGE plpgsql
+AS $$
+BEGIN
+
+  EXECUTE FORMAT (
+    'ALTER TABLE wire_targets_' || partition_suffix || ' '
+      'ADD CONSTRAINT wire_targets_' || partition_suffix || '_wire_target_serial_id_key '
+        'UNIQUE (wire_target_serial_id)'
+  );
+END
+$$;
+
+
+CREATE OR REPLACE FUNCTION detach_default_partitions2()
+  RETURNS VOID
+  LANGUAGE plpgsql
+AS $$
+BEGIN
+
+  RAISE NOTICE 'Detaching all default table partitions';
+
+  ALTER TABLE IF EXISTS wire_targets
+    DETACH PARTITION wire_targets_default;
+
+END
+$$;
+
+COMMENT ON FUNCTION detach_default_partitions2
+  IS 'We need to drop default and create new one before deleting the default partitions
+      otherwise constraints get lost too. Might be needed in sharding too';
+
+
+CREATE OR REPLACE FUNCTION drop_default_partitions2()
+  RETURNS VOID
+  LANGUAGE plpgsql
+AS $$
+BEGIN
+
+  RAISE NOTICE 'Dropping default table partitions';
+
+  DROP TABLE IF EXISTS wire_targets_default;
+END
+$$;
+
+COMMENT ON FUNCTION drop_default_partitions2
+  IS 'Drop all default partitions once other partitions are attached.
+      Might be needed in sharding too.';
+
+
+CREATE OR REPLACE FUNCTION create_partitions2(
+    num_partitions INTEGER
+)
+RETURNS VOID
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  modulus INTEGER;
+BEGIN
+
+  modulus := num_partitions;
+
+  PERFORM detach_default_partitions2();
+
+  LOOP
+
+    PERFORM create_hash_partition(
+      'wire_targets'
+      ,modulus
+      ,num_partitions
+    );
+    PERFORM add_constraints_to_wire_targets_partition(num_partitions::varchar);
+
+  END LOOP;
+
+  PERFORM drop_default_partitions2();
+
+END
+$$;
+
+
+CREATE OR REPLACE FUNCTION prepare_sharding2()
+RETURNS VOID
+LANGUAGE plpgsql
+AS $$
+BEGIN
+
+  PERFORM detach_default_partitions2();
+
+  ALTER TABLE IF EXISTS wire_targets
+    DROP CONSTRAINT IF EXISTS wire_targets_pkey CASCADE
+  ;
+
+END
+$$;
+
+
+CREATE OR REPLACE FUNCTION create_shard_server2(
+    shard_suffix VARCHAR
+    ,total_num_shards INTEGER
+    ,current_shard_num INTEGER
+    ,remote_host VARCHAR
+    ,remote_user VARCHAR
+    ,remote_user_password VARCHAR
+    ,remote_db_name VARCHAR DEFAULT 'taler-exchange'
+    ,remote_port INTEGER DEFAULT '5432'
+    ,local_user VARCHAR DEFAULT 'taler-exchange-httpd'
+)
+RETURNS VOID
+LANGUAGE plpgsql
+AS $$
+BEGIN
+
+  RAISE NOTICE 'Creating server %', remote_host;
+
+  PERFORM create_foreign_hash_partition(
+    'wire_targets'
+    ,total_num_shards
+    ,shard_suffix
+    ,current_shard_num
+    ,local_user
+  );
+END
+$$;
+
+COMMENT ON FUNCTION create_shard_server2
+  IS 'Create a shard server on the master
+      node with all foreign tables and user mappings';
+
--- a/src/exchangedb/exchange-0002-part.sql
+++ b/src/exchangedb/exchange-0002-part.sql
@ -0,0 +1,33 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2014--2022 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+--
+
+-- ------------------------------ wire_targets ----------------------------------------
+
+SELECT create_table_wire_targets();
+
+COMMENT ON TABLE wire_targets
+  IS 'All senders and recipients of money via the exchange';
+COMMENT ON COLUMN wire_targets.payto_uri
+  IS 'Can be a regular bank account, or also be a URI identifying a reserve-account (for P2P payments)';
+COMMENT ON COLUMN wire_targets.wire_target_h_payto
+  IS 'Unsalted hash of payto_uri';
+
+CREATE TABLE IF NOT EXISTS wire_targets_default
+  PARTITION OF wire_targets
+  FOR VALUES WITH (MODULUS 1, REMAINDER 0);
+
+SELECT add_constraints_to_wire_targets_partition('default');
+
--- a/src/exchangedb/exchange-0002.sql.in
+++ b/src/exchangedb/exchange-0002.sql.in
@ -0,0 +1,30 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2014--2022 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+--
+
+-- Everything in one big transaction
+BEGIN;
+
+-- Check patch versioning is in place.
+SELECT _v.register_patch('exchange-0002', NULL, NULL);
+
+-------------------- Schema ----------------------------
+
+SET search_path TO exchange;
+
+#include "common-0002.sql"
+#include "exchange-0002-part.sql"
+
+COMMIT;
--- a/src/exchangedb/shard-0002-part.sql
+++ b/src/exchangedb/shard-0002-part.sql
@ -0,0 +1,31 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2014--2022 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+--
+
+CREATE OR REPLACE FUNCTION setup_shard2(
+  shard_idx INTEGER
+)
+RETURNS VOID
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  shard_suffix VARCHAR;
+BEGIN
+
+  shard_suffix = shard_idx::varchar;
+
+  PERFORM create_table_wire_targets(shard_suffix);
+END
+$$;
--- a/src/exchangedb/shard-0002.sql.in
+++ b/src/exchangedb/shard-0002.sql.in
@ -0,0 +1,33 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2014--2022 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+--
+
+-- Everything in one big transaction
+BEGIN;
+
+-- Check patch versioning is in place.
+SELECT _v.register_patch('shard-0002', NULL, NULL);
+
+-------------------- Schema ----------------------------
+
+CREATE SCHEMA exchange;
+COMMENT ON SCHEMA exchange IS 'taler-exchange data';
+
+SET search_path TO exchange;
+
+#include "common-0002.sql"
+#include "shard-0002-part.sql"
+
+COMMIT;
--- a/src/exchangedb/test-exchange-db-postgres.conf
+++ b/src/exchangedb/test-exchange-db-postgres.conf
@ -7,7 +7,7 @@ BASE_URL = http://localhost/
 [exchangedb-postgres]

 #The connection string the plugin has to use for connecting to the database
-CONFIG = postgres://dab:test@localhost/talercheck
+CONFIG = postgres:///talercheck

 # Where are the SQL files to setup our tables?
 SQL_DIR = $DATADIR/sql/exchange/
@ -33,4 +33,4 @@ LEGAL_RESERVE_EXPIRATION_TIME = 7 years
 AGGREGATOR_SHIFT = 1s

 # Number of purses per account by default.
-DEFAULT_PURSE_LIMIT = 1
+DEFAULT_PURSE_LIMIT = 1