taler/exchange
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add helper logic for JSON2JSON conversion

Browse Source
This commit is contained in:
Christian Grothoff 2023-05-10 22:09:47 +02:00
parent 4954963405
commit b15713f42e
No known key found for this signature in database
GPG Key ID: 939E6BE1E29FC3CC
5 changed files with 621 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/include/taler_util.h
View File

@ -1,6 +1,6 @@
/*
This file is part of TALER
Copyright (C) 2014-2021 Taler Systems SA
Copyright (C) 2014-2023 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
@ -569,6 +569,58 @@ enum GNUNET_GenericReturnValue
TALER_JSON_parse_age_groups (const json_t *root,
struct TALER_AgeMask *mask);
/**
* Handle to an external process that will assist
* with some JSON-to-JSON conversion.
*/
struct TALER_JSON_ExternalConversion;
/**
* Type of a callback that receives a JSON @a result.
*
* @param cls closure
* @param status_type how did the process die
* @apram code termination status code from the process
* @param result some JSON result, NULL if we failed to get an JSON output
*/
typedef void
(*TALER_JSON_JsonCallback) (void *cls,
enum GNUNET_OS_ProcessStatusType status_type,
unsigned long code,
const json_t *result);
/**
* Launch some external helper @a binary to convert some @a input
* and eventually call @a cb with the result.
*
* @param input JSON to serialize and pass to the helper process
* @param cb function to call on the result
* @param cb_cls closure for @a cb
* @param binary name of the binary to execute
* @param ... NULL-terminated list of arguments for the @a binary,
* usually starting with again the name of the binary
* @return handle to cancel the operation (and kill the helper)
*/
struct TALER_JSON_ExternalConversion *
TALER_JSON_external_conversion_start (const json_t *input,
TALER_JSON_JsonCallback cb,
void *cb_cls,
const char *binary,
...);
/**
* Abort external conversion, killing the process and preventing
* the callback from being called. Must not be called after the
* callback was invoked.
*
* @param[in] ec external conversion handle to cancel
*/
void
TALER_JSON_external_conversion_stop (
struct TALER_JSON_ExternalConversion *ec);
#undef __TALER_UTIL_LIB_H_INSIDE__
#endif

13
src/util/Makefile.am
View File

@ -20,7 +20,8 @@ EXTRA_DIST = \
taler-config.in \
test_helper_eddsa.conf \
test_helper_rsa.conf \
test_helper_cs.conf
test_helper_cs.conf \
test_conversion.sh
bin_PROGRAMS = \
taler-exchange-secmod-eddsa \
@ -80,6 +81,7 @@ libtalerutil_la_SOURCES = \
aml_signatures.c \
auditor_signatures.c \
config.c \
conversion.c \
crypto.c \
crypto_confirmation.c \
crypto_contract.c \
@ -125,6 +127,7 @@ AM_TESTS_ENVIRONMENT=export TALER_PREFIX=$${TALER_PREFIX:-@libdir@};export PATH=
check_PROGRAMS = \
test_age_restriction \
test_amount \
test_conversion \
test_crypto \
test_helper_eddsa \
test_helper_rsa \
@ -141,6 +144,14 @@ test_age_restriction_LDADD = \
-lgnunetutil \
libtalerutil.la
test_conversion_SOURCES = \
test_conversion.c
test_conversion_LDADD = \
-lgnunetjson \
-lgnunetutil \
-ljansson \
libtalerutil.la
test_amount_SOURCES = \
test_amount.c
test_amount_LDADD = \

402
src/util/conversion.c Normal file
View File

@ -0,0 +1,402 @@
/*
This file is part of TALER
Copyright (C) 2023 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
TALER is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
*/
/**
* @file conversion.c
* @brief helper routines to run some external JSON-to-JSON converter
* @author Christian Grothoff
*/
#include "platform.h"
#include "taler_util.h"
#include <gnunet/gnunet_util_lib.h>
struct TALER_JSON_ExternalConversion
{
/**
* Callback to call with the result.
*/
TALER_JSON_JsonCallback cb;
/**
* Closure for @e cb.
*/
void *cb_cls;
/**
* Handle to the helper process.
*/
struct GNUNET_OS_Process *helper;
/**
* Pipe for the stdin of the @e helper.
*/
struct GNUNET_DISK_FileHandle *chld_stdin;
/**
* Pipe for the stdout of the @e helper.
*/
struct GNUNET_DISK_FileHandle *chld_stdout;
/**
* Handle to wait on the child to terminate.
*/
struct GNUNET_ChildWaitHandle *cwh;
/**
* Task to read JSON output from the child.
*/
struct GNUNET_SCHEDULER_Task *read_task;
/**
* Task to send JSON input to the child.
*/
struct GNUNET_SCHEDULER_Task *write_task;
/**
* Buffer with data we need to send to the helper.
*/
void *write_buf;
/**
* Buffer for reading data from the helper.
*/
void *read_buf;
/**
* Total length of @e write_buf.
*/
size_t write_size;
/**
* Current write position in @e write_buf.
*/
size_t write_pos;
/**
* Current size of @a read_buf.
*/
size_t read_size;
/**
* Current offset in @a read_buf.
*/
size_t read_pos;
};
/**
* Function called when we can read more data from
* the child process.
*
* @param cls our `struct TALER_JSON_ExternalConversion *`
*/
static void
read_cb (void *cls)
{
struct TALER_JSON_ExternalConversion *ec = cls;
ec->read_task = NULL;
while (1)
{
ssize_t ret;
if (ec->read_size == ec->read_pos)
{
/* Grow input buffer */
size_t ns;
void *tmp;
ns = GNUNET_MAX (2 * ec->read_size,
1024);
if (ns > GNUNET_MAX_MALLOC_CHECKED)
ns = GNUNET_MAX_MALLOC_CHECKED;
if (ec->read_size == ns)
{
/* Helper returned more than 40 MB of data! Stop reading! */
GNUNET_break (0);
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_file_close (ec->chld_stdin));
return;
}
tmp = GNUNET_malloc_large (ns);
if (NULL == tmp)
{
/* out of memory, also stop reading */
GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
"malloc");
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_file_close (ec->chld_stdin));
return;
}
GNUNET_memcpy (tmp,
ec->read_buf,
ec->read_pos);
GNUNET_free (ec->read_buf);
ec->read_buf = tmp;
ec->read_size = ns;
}
ret = GNUNET_DISK_file_read (ec->chld_stdout,
ec->read_buf,
ec->read_size - ec->read_pos);
if (ret < 0)
{
if ( (EAGAIN != errno) &&
(EWOULDBLOCK != errno) &&
(EINTR != errno) )
{
GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
"read");
return;
}
break;
}
if (0 == ret)
{
/* regular end of stream, good! */
return;
}
GNUNET_assert (ec->read_size >= ec->read_pos + ret);
ec->read_pos += ret;
}
ec->read_task
= GNUNET_SCHEDULER_add_read_file (
GNUNET_TIME_UNIT_FOREVER_REL,
ec->chld_stdout,
&read_cb,
ec);
}
/**
* Function called when we can write more data to
* the child process.
*
* @param cls our `struct TALER_JSON_ExternalConversion *`
*/
static void
write_cb (void *cls)
{
struct TALER_JSON_ExternalConversion *ec = cls;
ssize_t ret;
ec->write_task = NULL;
while (ec->write_size > ec->write_pos)
{
ret = GNUNET_DISK_file_write (ec->chld_stdin,
ec->write_buf + ec->write_pos,
ec->write_size - ec->write_pos);
if (ret < 0)
{
if ( (EAGAIN != errno) &&
(EINTR != errno) )
GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
"write");
break;
}
if (0 == ret)
{
GNUNET_break (0);
break;
}
GNUNET_assert (ec->write_size >= ec->write_pos + ret);
ec->write_pos += ret;
}
if ( (ec->write_size > ec->write_pos) &&
( (EAGAIN == errno) ||
(EWOULDBLOCK == errno) ||
(EINTR == errno) ) )
{
ec->write_task
= GNUNET_SCHEDULER_add_write_file (
GNUNET_TIME_UNIT_FOREVER_REL,
ec->chld_stdin,
&write_cb,
ec);
}
else
{
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_file_close (ec->chld_stdin));
ec->chld_stdin = NULL;
}
}
/**
* Defines a GNUNET_ChildCompletedCallback which is sent back
* upon death or completion of a child process.
*
* @param cls handle for the callback
* @param type type of the process
* @param exit_code status code of the process
*
*/
static void
child_done_cb (void *cls,
enum GNUNET_OS_ProcessStatusType type,
long unsigned int exit_code)
{
struct TALER_JSON_ExternalConversion *ec = cls;
json_t *j;
json_error_t err;
ec->cwh = NULL;
if (NULL != ec->read_task)
{
GNUNET_SCHEDULER_cancel (ec->read_task);
/* We could get the process termination notification before having drained
the read buffer. So drain it now, just in case. */
read_cb (ec);
}
if (NULL != ec->read_task)
{
GNUNET_SCHEDULER_cancel (ec->read_task);
ec->read_task = NULL;
}
GNUNET_OS_process_destroy (ec->helper);
ec->helper = NULL;
j = json_loadb (ec->read_buf,
ec->read_pos,
JSON_REJECT_DUPLICATES,
&err);
if (NULL == j)
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Failed to parse JSON from helper at %d: %s\n",
err.position,
err.text);
}
ec->cb (ec->cb_cls,
type,
exit_code,
j);
json_decref (j);
TALER_JSON_external_conversion_stop (ec);
}
struct TALER_JSON_ExternalConversion *
TALER_JSON_external_conversion_start (const json_t *input,
TALER_JSON_JsonCallback cb,
void *cb_cls,
const char *binary,
...)
{
struct TALER_JSON_ExternalConversion *ec;
struct GNUNET_DISK_PipeHandle *pipe_stdin;
struct GNUNET_DISK_PipeHandle *pipe_stdout;
va_list ap;
ec = GNUNET_new (struct TALER_JSON_ExternalConversion);
ec->cb = cb;
ec->cb_cls = cb_cls;
pipe_stdin = GNUNET_DISK_pipe (GNUNET_DISK_PF_BLOCKING_READ);
GNUNET_assert (NULL != pipe_stdin);
pipe_stdout = GNUNET_DISK_pipe (GNUNET_DISK_PF_BLOCKING_WRITE);
GNUNET_assert (NULL != pipe_stdout);
va_start (ap,
binary);
ec->helper = GNUNET_OS_start_process_va (GNUNET_OS_INHERIT_STD_ERR,
pipe_stdin,
pipe_stdout,
NULL,
binary,
ap);
va_end (ap);
if (NULL == ec->helper)
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Failed to run conversion helper `%s'\n",
binary);
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_pipe_close (pipe_stdin));
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_pipe_close (pipe_stdout));
GNUNET_free (ec);
return NULL;
}
ec->chld_stdin =
GNUNET_DISK_pipe_detach_end (pipe_stdin,
GNUNET_DISK_PIPE_END_WRITE);
ec->chld_stdout =
GNUNET_DISK_pipe_detach_end (pipe_stdout,
GNUNET_DISK_PIPE_END_READ);
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_pipe_close (pipe_stdin));
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_pipe_close (pipe_stdout));
ec->write_buf = json_dumps (input, JSON_COMPACT);
ec->write_size = strlen (ec->write_buf);
ec->read_task
= GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
ec->chld_stdout,
&read_cb,
ec);
ec->write_task
= GNUNET_SCHEDULER_add_write_file (GNUNET_TIME_UNIT_FOREVER_REL,
ec->chld_stdin,
&write_cb,
ec);
ec->cwh = GNUNET_wait_child (ec->helper,
&child_done_cb,
ec);
return ec;
}
void
TALER_JSON_external_conversion_stop (
struct TALER_JSON_ExternalConversion *ec)
{
if (NULL != ec->cwh)
{
GNUNET_wait_child_cancel (ec->cwh);
ec->cwh = NULL;
}
if (NULL != ec->helper)
{
GNUNET_break (0 ==
GNUNET_OS_process_kill (ec->helper,
SIGKILL));
GNUNET_OS_process_destroy (ec->helper);
}
if (NULL != ec->read_task)
{
GNUNET_SCHEDULER_cancel (ec->read_task);
ec->read_task = NULL;
}
if (NULL != ec->write_task)
{
GNUNET_SCHEDULER_cancel (ec->write_task);
ec->write_task = NULL;
}
if (NULL != ec->chld_stdin)
{
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_file_close (ec->chld_stdin));
ec->chld_stdin = NULL;
}
if (NULL != ec->chld_stdout)
{
GNUNET_break (GNUNET_OK ==
GNUNET_DISK_file_close (ec->chld_stdout));
ec->chld_stdout = NULL;
}
GNUNET_free (ec->read_buf);
free (ec->write_buf);
GNUNET_free (ec);
}

149
src/util/test_conversion.c Normal file
View File

@ -0,0 +1,149 @@
/*
This file is part of TALER
(C) 2023 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
TALER is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
*/
/**
* @file util/test_conversion.c
* @brief Tests for conversion logic
* @author Christian Grothoff
*/
#include "platform.h"
#include "taler_util.h"
#include <gnunet/gnunet_json_lib.h>
/**
* Return value from main().
*/
static int global_ret;
/**
* Handle to our helper.
*/
static struct TALER_JSON_ExternalConversion *ec;
/**
* Type of a callback that receives a JSON @a result.
*
* @param cls closure
* @param status_type how did the process die
* @apram code termination status code from the process
* @param result some JSON result, NULL if we failed to get an JSON output
*/
static void
conv_cb (void *cls,
enum GNUNET_OS_ProcessStatusType status_type,
unsigned long code,
const json_t *result)
{
json_t *expect;
(void) cls;
(void) status_type;
ec = NULL;
global_ret = 3;
if (42 != code)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Unexpected return value from helper: %u\n",
(unsigned int) code);
return;
}
expect = GNUNET_JSON_PACK (
GNUNET_JSON_pack_string ("foo",
"arg")
);
if (1 == json_equal (expect,
result))
{
global_ret = 0;
}
else
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Unexpected JSON result\n");
json_dumpf (result,
stderr,
JSON_INDENT (2));
global_ret = 4;
}
json_decref (expect);
}
/**
* Function called on shutdown/CTRL-C.
*
* @param cls NULL
*/
static void
do_shutdown (void *cls)
{
(void) cls;
if (NULL != ec)
{
GNUNET_break (0);
global_ret = 2;
TALER_JSON_external_conversion_stop (ec);
ec = NULL;
}
}
/**
* Main test function.
*
* @param cls NULL
*/
static void
run (void *cls)
{
json_t *input;
(void) cls;
GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
NULL);
input = GNUNET_JSON_PACK (
GNUNET_JSON_pack_string ("key",
"foo")
);
ec = TALER_JSON_external_conversion_start (input,
&conv_cb,
NULL,
"./test_conversion.sh",
"test_conversion.sh",
"arg",
NULL);
json_decref (input);
GNUNET_assert (NULL != ec);
}
int
main (int argc,
const char *const argv[])
{
(void) argc;
(void) argv;
unsetenv ("XDG_DATA_HOME");
unsetenv ("XDG_CONFIG_HOME");
GNUNET_log_setup ("test-conversion",
"WARNING",
NULL);
GNUNET_OS_init (TALER_project_data_default ());
global_ret = 1;
GNUNET_SCHEDULER_run (&run,
NULL);
return global_ret;
}

5
src/util/test_conversion.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
KEY=$(jq -r .key)
echo -n "{\"$KEY\":\"$1\"}"
exit 42