taler/exchange
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

-remove shard support

Browse Source
This commit is contained in:
Christian Grothoff 2022-11-26 23:26:44 +01:00
parent 95149f345f
commit 746a8a0cdb
No known key found for this signature in database
GPG Key ID: 939E6BE1E29FC3CC
16 changed files with 161 additions and 643 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/gana

@ -1 +1 @@
Subproject commit b971fd44ebf51a9c403dbbff144b43e73aa02c6c
Subproject commit a4a6b9ba4b2634c56194d53e36344686d7052cef

49
src/exchange-tools/taler-exchange-dbinit.c
View File

@ -49,21 +49,11 @@ static int gc_db;
*/
static uint32_t num_partitions;
/**
* -F option: setup a sharded database, i.e. create foreign tables/server
*/
static int shard_db;
/**
* -f option: force partitions to be created when there is only one
*/
static int force_create_partitions;
/**
* -S option: setup a database on a shard server, creates tables with suffix shard_idx
*/
static uint32_t shard_idx;
/**
* Main function that will be run.
*
@ -100,21 +90,6 @@ run (void *cls,
"Could not drop tables as requested. Either database was not yet initialized, or permission denied. Consult the logs. Will still try to create new tables.\n");
}
}
if (0 < shard_idx)
{
if (GNUNET_OK !=
plugin->create_shard_tables (plugin->cls,
shard_idx))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Could not create shard database\n");
global_ret = EXIT_NOTINSTALLED;
}
/* We do not want to continue if we are on a shard */
TALER_EXCHANGEDB_plugin_unload (plugin);
plugin = NULL;
return;
}
if (GNUNET_OK !=
plugin->create_tables (plugin->cls))
{
@ -132,16 +107,9 @@ run (void *cls,
&& force_create_partitions))
{
enum GNUNET_GenericReturnValue r = GNUNET_OK;
if (shard_db)
{
r = plugin->setup_foreign_servers (plugin->cls,
num_partitions);
}
else
{
r = plugin->setup_partitions (plugin->cls,
num_partitions);
}
r = plugin->setup_partitions (plugin->cls,
num_partitions);
if (GNUNET_OK != r)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@ -220,17 +188,8 @@ main (int argc,
GNUNET_GETOPT_option_uint ('P',
"partition",
"NUMBER",
"Setup a partitioned database where each table which can be partitioned holds NUMBER partitions on a single DB node (NOTE: sharding add -F for sharding)",
"Setup a partitioned database where each table which can be partitioned holds NUMBER partitions on a single DB node",
&num_partitions),
GNUNET_GETOPT_option_flag ('F',
"foreign",
"Setup a sharded database with foreign servers (shards) / tables rather than a partitioned one, must be called as DB superuser.",
&shard_db),
GNUNET_GETOPT_option_uint ('S',
"shard",
"INDEX",
"Setup a shard server, creates tables with INDEX as suffix",
&shard_idx),
GNUNET_GETOPT_option_flag ('f',
"force",
"Force partitions to be created if there is only one partition",

24
src/exchangedb/Makefile.am
View File

@ -18,32 +18,24 @@ sqlinputs = \
common-0001.sql \
exchange-0001.sql.in \
exchange-0001-part.sql \
shard-0001-part.sql \
shard-0001.sql.in \
common-0002.sql \
exchange-0002.sql.in \
exchange-0002-part.sql \
shard-0002-part.sql \
shard-0002.sql.in
exchange-0002-part.sql
sql_DATA = \
benchmark-0001.sql \
versioning.sql \
exchange-0001.sql \
drop.sql \
procedures.sql \
shard-0001.sql
procedures.sql
BUILT_SOURCES = \
shard-0001.sql \
exchange-0001.sql \
drop.sql \
procedures.sql
CLEANFILES = \
shard-0001.sql \
exchange-0001.sql \
shard-0002.sql \
exchange-0002.sql
exchange-0001.sql: common-0001.sql exchange-0001-part.sql exchange-0001.sql.in
@ -56,16 +48,6 @@ exchange-0002.sql: common-0002.sql exchange-0002-part.sql exchange-0002.sql.in
gcc -E -P -undef - < exchange-0002.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
chmod ugo-w $@
shard-0001.sql: common-0001.sql shard-0001-part.sql exchange-0001.sql.in
chmod +w $@ || true
gcc -E -P -undef - < shard-0001.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
chmod ugo-w $@
shard-0002.sql: common-0002.sql shard-0002-part.sql exchange-0002.sql.in
chmod +w $@ || true
gcc -E -P -undef - < shard-0002.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
chmod ugo-w $@
EXTRA_DIST = \
exchangedb.conf \
exchangedb-postgres.conf \
@ -111,7 +93,6 @@ libtaler_plugin_exchangedb_postgres_la_SOURCES = \
pg_select_contract_by_purse.h pg_select_contract_by_purse.c \
pg_insert_drain_profit.h pg_insert_drain_profit.c \
pg_create_tables.h pg_create_tables.c \
pg_setup_foreign_servers.h pg_setup_foreign_servers.c \
pg_event_listen.h pg_event_listen.c \
pg_event_listen_cancel.h pg_event_listen_cancel.c \
pg_event_notify.h pg_event_notify.c \
@ -225,7 +206,6 @@ libtaler_plugin_exchangedb_postgres_la_SOURCES = \
pg_do_batch_withdraw_insert.h pg_do_batch_withdraw_insert.c \
pg_do_reserve_open.c pg_do_reserve_open.h \
pg_do_withdraw.h pg_do_withdraw.c \
pg_create_shard_tables.h pg_create_shard_tables.c \
pg_preflight.h pg_preflight.c \
pg_iterate_active_signkeys.h pg_iterate_active_signkeys.c \
pg_commit.h pg_commit.c \

81
src/exchangedb/common-0001.sql
View File

@ -50,7 +50,6 @@ RETURNS VOID
LANGUAGE plpgsql
AS $$
BEGIN
PERFORM create_partitioned_table(
'CREATE TABLE IF NOT EXISTS %I'
'(wire_target_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY' -- UNIQUE'
@ -61,7 +60,6 @@ BEGIN
,'PARTITION BY HASH (wire_target_h_payto)'
,shard_suffix
);
END
$$;
@ -1010,61 +1008,6 @@ BEGIN
END
$$;
----------------------------- refunds ------------------------------
CREATE OR REPLACE FUNCTION create_table_refunds(
IN shard_suffix VARCHAR DEFAULT NULL
)
RETURNS VOID
LANGUAGE plpgsql
AS $$
DECLARE
table_name VARCHAR DEFAULT 'refunds';
BEGIN
PERFORM create_partitioned_table(
'CREATE TABLE IF NOT EXISTS %I'
'(refund_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY' -- UNIQUE'
',coin_pub BYTEA NOT NULL CHECK (LENGTH(coin_pub)=32)' -- REFERENCES known_coins (coin_pub) ON DELETE CASCADE
',deposit_serial_id INT8 NOT NULL' -- REFERENCES deposits (deposit_serial_id) ON DELETE CASCADE'
',merchant_sig BYTEA NOT NULL CHECK(LENGTH(merchant_sig)=64)'
',rtransaction_id INT8 NOT NULL'
',amount_with_fee_val INT8 NOT NULL'
',amount_with_fee_frac INT4 NOT NULL'
-- ,PRIMARY KEY (deposit_serial_id, rtransaction_id) -- done per shard!
') %s ;'
,table_name
,'PARTITION BY HASH (coin_pub)'
,shard_suffix
);
table_name = concat_ws('_', table_name, shard_suffix);
EXECUTE FORMAT (
'CREATE INDEX IF NOT EXISTS ' || table_name || '_by_coin_pub_index '
'ON ' || table_name || ' '
'(coin_pub);'
);
END
$$;
CREATE OR REPLACE FUNCTION add_constraints_to_refunds_partition(
IN partition_suffix VARCHAR
)
RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
EXECUTE FORMAT (
'ALTER TABLE refunds_' || partition_suffix || ' '
'ADD CONSTRAINT refunds_' || partition_suffix || '_refund_serial_id_key '
'UNIQUE (refund_serial_id) '
',ADD PRIMARY KEY (deposit_serial_id, rtransaction_id) '
);
END
$$;
---------------------------- wire_out -------------------------------
CREATE OR REPLACE FUNCTION create_table_wire_out(
@ -2028,30 +1971,6 @@ $$;
------------------------- Partitions ------------------------------
-------------------------------------------------------------------
CREATE OR REPLACE FUNCTION create_hash_partition(
source_table_name VARCHAR
,modulus INTEGER
,partition_num INTEGER
)
RETURNS VOID
LANGUAGE plpgsql
AS $$
BEGIN
RAISE NOTICE 'Creating partition %_%', source_table_name, partition_num;
EXECUTE FORMAT(
'CREATE TABLE IF NOT EXISTS %I '
'PARTITION OF %I '
'FOR VALUES WITH (MODULUS %s, REMAINDER %s)'
,source_table_name || '_' || partition_num
,source_table_name
,modulus
,partition_num-1
);
END
$$;
CREATE OR REPLACE FUNCTION create_range_partition(
source_table_name VARCHAR

288
src/exchangedb/common-0002.sql
View File

@ -14,192 +14,174 @@
-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
--
--------------------------------
CREATE TABLE IF NOT EXISTS exchange_tables
(table_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY
,name VARCHAR NOT NULL
,version VARCHAR NOT NULL
,action VARCHAR NOT NULL
,partitioned BOOL NOT NULL
,by_range BOOL NOT NULL);
COMMENT ON TABLE exchange_tables
IS 'Tables of the exchange and their status';
COMMENT ON COLUMN exchange_tables.name
IS 'Base name of the table (without partition/shard)';
COMMENT ON COLUMN exchange_tables.version
IS 'Version of the DB in which the given action happened';
COMMENT ON COLUMN exchange_tables.action
IS 'Action to take on the table (e.g. create, alter, constrain, unconstrain, or drop)';
COMMENT ON COLUMN exchange_tables.partitioned
IS 'TRUE if the table is partitioned';
COMMENT ON COLUMN exchange_tables.by_range
IS 'TRUE if the table is partitioned by range';
INSERT INTO exchange_tables
(name
,version
,action
,partitioned
,by_range)
VALUES
('denominations', 'exchange-0001', 'create', FALSE, FALSE)
,('denomination_revocations', 'exchange-0001', 'create', FALSE, FALSE)
,('kyc_alerts', 'exchange-0001', 'create', FALSE, FALSE)
,('profit_drains', 'exchange-0001', 'create', FALSE, FALSE)
,('auditors', 'exchange-0001', 'create', FALSE, FALSE)
,('auditor_denom_sigs', 'exchange-0001', 'create', FALSE, FALSE)
,('exchange_sign_keys', 'exchange-0001', 'create', FALSE, FALSE)
,('signkey_revocations', 'exchange-0001', 'create', FALSE, FALSE)
,('extensions', 'exchange-0001', 'create', FALSE, FALSE)
,('wire_fee', 'exchange-0001', 'create', FALSE, FALSE)
,('global_fee', 'exchange-0001', 'create', FALSE, FALSE)
,('wire_accounts', 'exchange-0001', 'create', FALSE, FALSE)
,('work_shards', 'exchange-0001', 'create', FALSE, FALSE)
,('revolving_work_shards', 'exchange-0001', 'create', FALSE, FALSE)
,('partners', 'exchange-0001', 'create', FALSE, FALSE)
,('partner_accounts', 'exchange-0001', 'create', FALSE, FALSE)
,('purse_actions', 'exchange-0001', 'create', FALSE, FALSE)
,('policy_fulfillments', 'exchange-0001', 'create', FALSE, FALSE) -- bad!
,('policy_details', 'exchange-0001', 'create', FALSE, FALSE) -- bad!
,('wire_targets''exchange-0001', 'create', TRUE, FALSE)
,('legitimization_processes', 'exchange-0001', 'create', TRUE, FALSE)
,('legitimization_requirements', 'exchange-0001', 'create', TRUE, FALSE)
,('reserves', 'exchange-0001', 'create', TRUE, FALSE)
,('reserves_in', 'exchange-0001', 'create', TRUE, FALSE)
,('reserves_close', 'exchange-0001', 'create', TRUE, FALSE)
,('reserves_open_requests', 'exchange-0001', 'create', TRUE, FALSE)
,('reserves_open_deposits', 'exchange-0001', 'create', TRUE, FALSE)
,('reserves_out', 'exchange-0001', 'create', TRUE, FALSE)
,('reserves_out_by_reserve', 'exchange-0001', 'create', TRUE, FALSE)
,('known_coins', 'exchange-0001', 'create', TRUE, FALSE)
,('refresh_commitments', 'exchange-0001', 'create', TRUE, FALSE)
,('refresh_revealed_coins', 'exchange-0001', 'create', TRUE, FALSE)
,('refresh_transfer_keys', 'exchange-0001', 'create', TRUE, FALSE)
,('refunds', 'exchange-0001', 'create', TRUE, FALSE)
,('deposits', 'exchange-0001', 'create', TRUE, FALSE)
,('deposits_by_ready', 'exchange-0001', 'create', TRUE, TRUE)
,('deposits_for_matching', 'exchange-0001', 'create', TRUE, TRUE)
,('wire_out', 'exchange-0001', 'create', TRUE, FALSE)
,('aggregation_transient', 'exchange-0001', 'create', TRUE, FALSE)
,('aggregation_tracking', 'exchange-0001', 'create', TRUE, FALSE)
,('recoup', 'exchange-0001', 'create', TRUE, FALSE)
,('recoup_by_reserve', 'exchange-0001', 'create', TRUE, FALSE)
,('recoup_refresh', 'exchange-0001', 'create', TRUE, FALSE)
,('prewire', 'exchange-0001', 'create', TRUE, FALSE)
,('cs_nonce_locks', 'exchange-0001', 'create', TRUE, FALSE)
,('purse_requests', 'exchange-0001', 'create', TRUE, FALSE)
,('purse_decision', 'exchange-0001', 'create', TRUE, FALSE)
,('purse_merges', 'exchange-0001', 'create', TRUE, FALSE)
,('account_merges', 'exchange-0001', 'create', TRUE, FALSE)
,('contracts', 'exchange-0001', 'create', TRUE, FALSE)
,('history_requests', 'exchange-0001', 'create', TRUE, FALSE)
,('close_requests', 'exchange-0001', 'create', TRUE, FALSE)
,('purse_deposists', 'exchange-0001', 'create', TRUE, FALSE)
,('wads_out', 'exchange-0001', 'create', TRUE, FALSE)
,('wads_out_entries', 'exchange-0001', 'create', TRUE, FALSE)
,('wads_in', 'exchange-0001', 'create', TRUE, FALSE)
,('wads_in_entries', 'exchange-0001', 'create', TRUE, FALSE)
ON CONFLICT DO NOTHING;
-------------------- Tables ----------------------------
CREATE OR REPLACE FUNCTION create_table_wire_targets(
CREATE OR REPLACE FUNCTION create_table_purse_deletion(
IN shard_suffix VARCHAR DEFAULT NULL
)
RETURNS VOID
LANGUAGE plpgsql
AS $$
BEGIN
PERFORM create_partitioned_table(
'CREATE TABLE IF NOT EXISTS %I'
'(wire_target_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY' -- UNIQUE'
',wire_target_h_payto BYTEA PRIMARY KEY CHECK (LENGTH(wire_target_h_payto)=32)'
',payto_uri VARCHAR NOT NULL'
'(purse_deletion_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY' -- UNIQUE'
',purse_sig BYTEA CHECK (LENGTH(purse_sig)=64)'
',XXX VARCHAR NOT NULL'
') %s ;'
,'wire_targets'
,'PARTITION BY HASH (wire_target_h_payto)'
,'purse_deletion'
,'PARTITION BY HASH (XXX)'
,shard_suffix
);
COMMENT ON TABLE purse_deletion
IS 'signatures affirming explicit purse deletions';
COMMENT ON COLUMN purse_deletion.purse_sig
IS 'signature of type XXX';
END
$$;
COMMENT ON FUNCTION create_table_purse_deletion
IS 'Creates the purse_deletion table';
-- We need a separate function for this, as we call create_table only once but need to add
-- those constraints to each partition which gets created
CREATE OR REPLACE FUNCTION add_constraints_to_wire_targets_partition(
CREATE OR REPLACE FUNCTION constrain_table_purse_deletion(
IN partition_suffix VARCHAR
)
RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
EXECUTE FORMAT (
'ALTER TABLE wire_targets_' || partition_suffix || ' '
'ADD CONSTRAINT wire_targets_' || partition_suffix || '_wire_target_serial_id_key '
'UNIQUE (wire_target_serial_id)'
'ALTER TABLE purse_deletion_' || partition_suffix || ' '
'ADD CONSTRAINT purse_deletion_' || partition_suffix || '_XXX '
'UNIQUE (XXX)'
);
END
$$;
CREATE OR REPLACE FUNCTION detach_default_partitions2()
RETURNS VOID
LANGUAGE plpgsql
AS $$
BEGIN
RAISE NOTICE 'Detaching all default table partitions';
ALTER TABLE IF EXISTS wire_targets
DETACH PARTITION wire_targets_default;
END
$$;
COMMENT ON FUNCTION detach_default_partitions2
IS 'We need to drop default and create new one before deleting the default partitions
otherwise constraints get lost too. Might be needed in sharding too';
CREATE OR REPLACE FUNCTION drop_default_partitions2()
RETURNS VOID
LANGUAGE plpgsql
AS $$
BEGIN
RAISE NOTICE 'Dropping default table partitions';
DROP TABLE IF EXISTS wire_targets_default;
END
$$;
COMMENT ON FUNCTION drop_default_partitions2
IS 'Drop all default partitions once other partitions are attached.
Might be needed in sharding too.';
CREATE OR REPLACE FUNCTION create_partitions2(
num_partitions INTEGER
CREATE OR REPLACE FUNCTION alter_create_was_deleted_table_purse_requests (
IN partition_suffix VARCHAR
)
RETURNS VOID
LANGUAGE plpgsql
AS $$
DECLARE
modulus INTEGER;
BEGIN
modulus := num_partitions;
PERFORM detach_default_partitions2();
LOOP
PERFORM create_hash_partition(
'wire_targets'
,modulus
,num_partitions
);
PERFORM add_constraints_to_wire_targets_partition(num_partitions::varchar);
END LOOP;
PERFORM drop_default_partitions2();
END
$$;
CREATE OR REPLACE FUNCTION prepare_sharding2()
RETURNS VOID
RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
PERFORM detach_default_partitions2();
ALTER TABLE IF EXISTS wire_targets
DROP CONSTRAINT IF EXISTS wire_targets_pkey CASCADE
;
END
$$;
CREATE OR REPLACE FUNCTION create_shard_server2(
shard_suffix VARCHAR
,total_num_shards INTEGER
,current_shard_num INTEGER
,remote_host VARCHAR
,remote_user VARCHAR
,remote_user_password VARCHAR
,remote_db_name VARCHAR DEFAULT 'taler-exchange'
,remote_port INTEGER DEFAULT '5432'
,local_user VARCHAR DEFAULT 'taler-exchange-httpd'
)
RETURNS VOID
LANGUAGE plpgsql
AS $$
BEGIN
RAISE NOTICE 'Creating server %', remote_host;
PERFORM create_foreign_hash_partition(
'wire_targets'
,total_num_shards
,shard_suffix
,current_shard_num
,local_user
EXECUTE FORMAT (
'ALTER TABLE purse_requests_' || partition_suffix ||
' ADD COLUMN'
' was_deleted BOOLEAN NOT NULL DEFAULT(FALSE)'
);
END
$$;
COMMENT ON FUNCTION create_shard_server2
IS 'Create a shard server on the master
node with all foreign tables and user mappings';
--------------------------------
CREATE TABLE IF NOT EXISTS partitioned_tables
(name VARCHAR PRIMARY KEY NOT NULL);
INSERT INTO partitioned_tables
(name)
INSERT INTO exchange_tables
(name
,version
,action
,partitioned
,by_range)
VALUES
('wire_targets')
,('refunds')
ON CONFLICT DO NOTHING;
CREATE OR REPLACE FUNCTION drop_default_partitions()
RETURNS VOID
LANGUAGE plpgsql
AS $$
DECLARE
tc CURSOR FOR SELECT name FROM partitioned_tables;
BEGIN
RAISE NOTICE 'Dropping default table partitions';
FOR rec IN tc
LOOP
EXECUTE FORMAT (
'DROP TABLE IF EXISTS %s_default ;'::text,
rec.name;
END
$$;
('purse_deletion'
,'exchange-0002'
,'create'
,TRUE
,FALSE),
('purse_deletion'
,'exchange-0002'
,'constrain'
,TRUE
,FALSE),
('purse_requests'
,'exchange-0002'
,'alter_create_was_deleted'
,TRUE
,FALSE);

34
src/exchangedb/exchange-0001-part.sql
View File

@ -16,40 +16,6 @@
-- ------------------------------ denominations ----------------------------------------
CREATE TABLE IF NOT EXISTS denominations
(denominations_serial BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE
,denom_pub_hash BYTEA PRIMARY KEY CHECK (LENGTH(denom_pub_hash)=64)
,denom_type INT4 NOT NULL DEFAULT (1) -- 1 == RSA (for now, remove default later!)
,age_mask INT4 NOT NULL DEFAULT (0)
,denom_pub BYTEA NOT NULL
,master_sig BYTEA NOT NULL CHECK (LENGTH(master_sig)=64)
,valid_from INT8 NOT NULL
,expire_withdraw INT8 NOT NULL
,expire_deposit INT8 NOT NULL
,expire_legal INT8 NOT NULL
,coin_val INT8 NOT NULL
,coin_frac INT4 NOT NULL
,fee_withdraw_val INT8 NOT NULL
,fee_withdraw_frac INT4 NOT NULL
,fee_deposit_val INT8 NOT NULL
,fee_deposit_frac INT4 NOT NULL
,fee_refresh_val INT8 NOT NULL
,fee_refresh_frac INT4 NOT NULL
,fee_refund_val INT8 NOT NULL
,fee_refund_frac INT4 NOT NULL
);
COMMENT ON TABLE denominations
IS 'Main denominations table. All the valid denominations the exchange knows about.';
COMMENT ON COLUMN denominations.denom_type
IS 'determines cipher type for blind signatures used with this denomination; 0 is for RSA';
COMMENT ON COLUMN denominations.age_mask
IS 'bitmask with the age restrictions that are being used for this denomination; 0 if denomination does not support the use of age restrictions';
COMMENT ON COLUMN denominations.denominations_serial
IS 'needed for exchange-auditor replication logic';
CREATE INDEX IF NOT EXISTS denominations_by_expire_legal_index
ON denominations
(expire_legal);
-- ------------------------------ denomination_revocations ----------------------------------------

3
src/exchangedb/exchange-0002-part.sql
View File

@ -31,3 +31,6 @@ CREATE TABLE IF NOT EXISTS wire_targets_default
SELECT add_constraints_to_wire_targets_partition('default');
SELECT create_tables('exchange-0002');

3
src/exchangedb/exchange-0002.sql.in
View File

@ -24,7 +24,6 @@ SELECT _v.register_patch('exchange-0002', NULL, NULL);
SET search_path TO exchange;
#include "common-0002.sql"
#include "exchange-0002-part.sql"
#include "0002-denominations.sql"
COMMIT;

9
src/exchangedb/pg_complete_shard.c
View File

@ -27,12 +27,11 @@
enum GNUNET_DB_QueryStatus
TEH_PG_complete_shard (void *cls,
const char *job_name,
uint64_t start_row,
uint64_t end_row)
const char *job_name,
uint64_t start_row,
uint64_t end_row)
{
struct PostgresClosure *pg = cls;
struct GNUNET_PQ_QueryParam params[] = {
GNUNET_PQ_query_param_string (job_name),
GNUNET_PQ_query_param_uint64 (&start_row),
@ -44,8 +43,6 @@ TEH_PG_complete_shard (void *cls,
"Completing shard %llu-%llu\n",
(unsigned long long) start_row,
(unsigned long long) end_row);
PREPARE (pg,
"complete_shard",
"UPDATE work_shards"

66
src/exchangedb/pg_create_shard_tables.c
View File

@ -1,66 +0,0 @@
/*
This file is part of TALER
Copyright (C) 2022 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
TALER is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
*/
/**
* @file exchangedb/pg_create_shard_tables.c
* @brief Implementation of the create_shard_tables function for Postgres
* @author Christian Grothoff
*/
#include "platform.h"
#include "taler_error_codes.h"
#include "taler_dbevents.h"
#include "taler_pq_lib.h"
#include "pg_create_shard_tables.h"
#include "pg_helper.h"
enum GNUNET_GenericReturnValue
TEH_PG_create_shard_tables (void *cls,
uint32_t idx)
{
struct PostgresClosure *pg = cls;
struct GNUNET_PQ_Context *conn;
enum GNUNET_GenericReturnValue ret = GNUNET_OK;
struct GNUNET_PQ_QueryParam params[] = {
GNUNET_PQ_query_param_uint32 (&idx),
GNUNET_PQ_query_param_end
};
struct GNUNET_PQ_ExecuteStatement es[] = {
GNUNET_PQ_make_try_execute ("SET search_path TO exchange;"),
GNUNET_PQ_EXECUTE_STATEMENT_END
};
struct GNUNET_PQ_PreparedStatement ps[] = {
GNUNET_PQ_make_prepare ("create_shard_tables",
"SELECT"
" setup_shard"
" ($1);"),
GNUNET_PQ_PREPARED_STATEMENT_END
};
conn = GNUNET_PQ_connect_with_cfg (pg->cfg,
"exchangedb-postgres",
"shard-",
es,
ps);
if (NULL == conn)
return GNUNET_SYSERR;
if (0 > GNUNET_PQ_eval_prepared_non_select (conn,
"create_shard_tables",
params))
ret = GNUNET_SYSERR;
GNUNET_PQ_disconnect (conn);
return ret;
}

39
src/exchangedb/pg_create_shard_tables.h
View File

@ -1,39 +0,0 @@
/*
This file is part of TALER
Copyright (C) 2022 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
TALER is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
*/
/**
* @file exchangedb/pg_create_shard_tables.h
* @brief implementation of the create_shard_tables function for Postgres
* @author Christian Grothoff
*/
#ifndef PG_CREATE_SHARD_TABLES_H
#define PG_CREATE_SHARD_TABLES_H
#include "taler_util.h"
#include "taler_json_lib.h"
#include "taler_exchangedb_plugin.h"
/**
* Create tables of a shard node with index idx
*
* @param cls the `struct PostgresClosure` with the plugin-specific state
* @param idx the shards index, will be appended as suffix to all tables
* @return #GNUNET_OK upon success; #GNUNET_SYSERR upon failure
*/
enum GNUNET_GenericReturnValue
TEH_PG_create_shard_tables (void *cls,
uint32_t idx);
#endif

118
src/exchangedb/pg_setup_foreign_servers.c
View File

@ -1,118 +0,0 @@
/*
This file is part of TALER
Copyright (C) 2022 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
TALER is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
*/
/**
* @file exchangedb/pg_setup_foreign_servers.c
* @brief Implementation of the setup_foreign_servers function for Postgres
* @author Christian Grothoff
*/
#include "platform.h"
#include "taler_error_codes.h"
#include "taler_dbevents.h"
#include "taler_pq_lib.h"
#include "pg_setup_foreign_servers.h"
#include "pg_helper.h"
enum GNUNET_GenericReturnValue
TEH_PG_setup_foreign_servers (void *cls,
uint32_t num)
{
struct PostgresClosure *pg = cls;
struct GNUNET_PQ_Context *conn;
enum GNUNET_GenericReturnValue ret = GNUNET_OK;
char *shard_domain = NULL;
char *remote_user = NULL;
char *remote_user_pw = NULL;
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_string (pg->cfg,
"exchange",
"SHARD_DOMAIN",
&shard_domain))
{
GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
"exchange",
"SHARD_DOMAIN");
return GNUNET_SYSERR;
}
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_string (pg->cfg,
"exchangedb-postgres",
"SHARD_REMOTE_USER",
&remote_user))
{
GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
"exchangedb-postgres",
"SHARD_REMOTE_USER");
GNUNET_free (shard_domain);
return GNUNET_SYSERR;
}
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_string (pg->cfg,
"exchangedb-postgres",
"SHARD_REMOTE_USER_PW",
&remote_user_pw))
{
GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
"exchangedb-postgres",
"SHARD_REMOTE_USER_PW");
GNUNET_free (shard_domain);
GNUNET_free (remote_user);
return GNUNET_SYSERR;
}
struct GNUNET_PQ_QueryParam params[] = {
GNUNET_PQ_query_param_uint32 (&num),
GNUNET_PQ_query_param_string (shard_domain),
GNUNET_PQ_query_param_string (remote_user),
GNUNET_PQ_query_param_string (remote_user_pw),
GNUNET_PQ_query_param_end
};
struct GNUNET_PQ_ExecuteStatement es[] = {
GNUNET_PQ_make_try_execute ("SET search_path TO exchange;"),
GNUNET_PQ_EXECUTE_STATEMENT_END
};
struct GNUNET_PQ_PreparedStatement ps[] = {
GNUNET_PQ_make_prepare ("create_foreign_servers",
"SELECT"
" create_foreign_servers"
" ($1, $2, $3, $4);"),
GNUNET_PQ_PREPARED_STATEMENT_END
};
conn = GNUNET_PQ_connect_with_cfg (pg->cfg,
"exchangedb-postgres",
NULL,
es,
ps);
if (NULL == conn)
{
ret = GNUNET_SYSERR;
}
else if (0 > GNUNET_PQ_eval_prepared_non_select (conn,
"create_foreign_servers",
params))
{
ret = GNUNET_SYSERR;
}
GNUNET_free (shard_domain);
GNUNET_free (remote_user);
GNUNET_free (remote_user_pw);
GNUNET_PQ_disconnect (conn);
return ret;
}

39
src/exchangedb/pg_setup_foreign_servers.h
View File

@ -1,39 +0,0 @@
/*
This file is part of TALER
Copyright (C) 2022 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
TALER is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
*/
/**
* @file exchangedb/pg_setup_foreign_servers.h
* @brief implementation of the setup_foreign_servers function for Postgres
* @author Christian Grothoff
*/
#ifndef PG_SETUP_FOREIGN_SERVERS_H
#define PG_SETUP_FOREIGN_SERVERS_H
#include "taler_util.h"
#include "taler_json_lib.h"
#include "taler_exchangedb_plugin.h"
/**
* Setup foreign servers (shards) for already existing tables
*
* @param cls the `struct PostgresClosure` with the plugin-specific state
* @param num the number of foreign servers (shards) to create for each partitioned table
* @return #GNUNET_OK upon success; #GNUNET_SYSERR upon failure
*/
enum GNUNET_GenericReturnValue
TEH_PG_setup_foreign_servers (void *cls,
uint32_t num);
#endif

30
src/exchangedb/plugin_exchangedb_postgres.c
View File

@ -65,10 +65,8 @@
#include "pg_iterate_active_signkeys.h"
#include "pg_preflight.h"
#include "pg_commit.h"
#include "pg_create_shard_tables.h"
#include "pg_insert_aggregation_tracking.h"
#include "pg_drop_tables.h"
#include "pg_setup_partitions.h"
#include "pg_select_satisfied_kyc_processes.h"
#include "pg_select_aggregation_amounts_for_kyc_check.h"
#include "pg_kyc_provider_account_lookup.h"
@ -123,6 +121,7 @@
#include "pg_get_policy_details.h"
#include "pg_persist_policy_details.h"
#include "pg_do_deposit.h"
#include "pg_setup_partitions.h"
#include "pg_add_policy_fulfillment_proof.h"
#include "pg_do_melt.h"
#include "pg_do_refund.h"
@ -384,9 +383,6 @@ TEH_PG_internal_setup (struct PostgresClosure *pg,
}
/**
* Closure for #get_refunds_cb().
*/
@ -456,9 +452,6 @@ get_refunds_cb (void *cls,
}
/* Get the details of a policy, referenced by its hash code
*
* @param cls the `struct PostgresClosure` with the plugin-specific state
@ -1627,8 +1620,9 @@ postgres_ensure_coin_known (void *cls,
return TALER_EXCHANGEDB_CKS_PRESENT;
}
enum GNUNET_DB_QueryStatus
setup_wire_target(
setup_wire_target (
struct PostgresClosure *pg,
const char *payto_uri,
struct TALER_PaytoHashP *h_payto)
@ -1654,6 +1648,8 @@ setup_wire_target(
"insert_kyc_status",
iparams);
}
/**
* Insert information about deposited coin into the database.
*
@ -2866,7 +2862,7 @@ postgres_insert_reserve_closed (
reserve.pub = *reserve_pub;
if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
(qs = TEH_PG_reserves_get (cls,
&reserve)))
&reserve)))
{
/* Existence should have been checked before we got here... */
GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
@ -2893,7 +2889,7 @@ postgres_insert_reserve_closed (
GNUNET_break (TALER_AAR_RESULT_ZERO == ret);
}
return TEH_PG_reserves_update (cls,
&reserve);
&reserve);
}
@ -5168,12 +5164,8 @@ libtaler_plugin_exchangedb_postgres_init (void *cls)
= &TEH_PG_commit;
plugin->preflight
= &TEH_PG_preflight;
plugin->create_shard_tables
= &TEH_PG_create_shard_tables;
plugin->insert_aggregation_tracking
= &TEH_PG_insert_aggregation_tracking;
plugin->setup_partitions
= &TEH_PG_setup_partitions;
plugin->select_aggregation_amounts_for_kyc_check
= &TEH_PG_select_aggregation_amounts_for_kyc_check;
@ -5245,11 +5237,8 @@ libtaler_plugin_exchangedb_postgres_init (void *cls)
= &TEH_PG_start;
plugin->rollback
= &TEH_PG_rollback;
plugin->create_tables
plugin->create_tables
= &TEH_PG_create_tables;
plugin->setup_foreign_servers
= &TEH_PG_setup_foreign_servers;
plugin->event_listen
= &TEH_PG_event_listen;
plugin->event_listen_cancel
@ -5446,7 +5435,8 @@ libtaler_plugin_exchangedb_postgres_init (void *cls)
= &TEH_PG_select_purse_by_merge_pub;
plugin->set_purse_balance
= &TEH_PG_set_purse_balance;
plugin->setup_partitions
= &TEH_PG_setup_partitions;
plugin->batch_reserves_in_insert
= &TEH_PG_batch_reserves_in_insert;

1
src/exchangedb/procedures.sql
View File

@ -23,6 +23,7 @@ SET search_path TO exchange;
-- Stored procedures
---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION exchange_do_withdraw(
IN cs_nonce BYTEA,
IN amount_val INT8,

16
src/exchangedb/shard-0002-part.sql
View File

@ -13,19 +13,3 @@
-- You should have received a copy of the GNU General Public License along with
-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
--
CREATE OR REPLACE FUNCTION setup_shard2(
shard_idx INTEGER
)
RETURNS VOID
LANGUAGE plpgsql
AS $$
DECLARE
shard_suffix VARCHAR;
BEGIN
shard_suffix = shard_idx::varchar;
PERFORM create_table_wire_targets(shard_suffix);
END
$$;