diff --git a/src/include/taler_util.h b/src/include/taler_util.h
index 6c2948221..0b15eb8a8 100644
--- a/src/include/taler_util.h
+++ b/src/include/taler_util.h
@@ -1,6 +1,6 @@
/*
This file is part of TALER
- Copyright (C) 2014-2021 Taler Systems SA
+ Copyright (C) 2014-2023 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
@@ -569,6 +569,58 @@ enum GNUNET_GenericReturnValue
TALER_JSON_parse_age_groups (const json_t *root,
struct TALER_AgeMask *mask);
+
+/**
+ * Handle to an external process that will assist
+ * with some JSON-to-JSON conversion.
+ */
+struct TALER_JSON_ExternalConversion;
+
+/**
+ * Type of a callback that receives a JSON @a result.
+ *
+ * @param cls closure
+ * @param status_type how did the process die
+ * @apram code termination status code from the process
+ * @param result some JSON result, NULL if we failed to get an JSON output
+ */
+typedef void
+(*TALER_JSON_JsonCallback) (void *cls,
+ enum GNUNET_OS_ProcessStatusType status_type,
+ unsigned long code,
+ const json_t *result);
+
+
+/**
+ * Launch some external helper @a binary to convert some @a input
+ * and eventually call @a cb with the result.
+ *
+ * @param input JSON to serialize and pass to the helper process
+ * @param cb function to call on the result
+ * @param cb_cls closure for @a cb
+ * @param binary name of the binary to execute
+ * @param ... NULL-terminated list of arguments for the @a binary,
+ * usually starting with again the name of the binary
+ * @return handle to cancel the operation (and kill the helper)
+ */
+struct TALER_JSON_ExternalConversion *
+TALER_JSON_external_conversion_start (const json_t *input,
+ TALER_JSON_JsonCallback cb,
+ void *cb_cls,
+ const char *binary,
+ ...);
+
+/**
+ * Abort external conversion, killing the process and preventing
+ * the callback from being called. Must not be called after the
+ * callback was invoked.
+ *
+ * @param[in] ec external conversion handle to cancel
+ */
+void
+TALER_JSON_external_conversion_stop (
+ struct TALER_JSON_ExternalConversion *ec);
+
#undef __TALER_UTIL_LIB_H_INSIDE__
#endif
diff --git a/src/util/Makefile.am b/src/util/Makefile.am
index 115ea5055..9d1ec9d42 100644
--- a/src/util/Makefile.am
+++ b/src/util/Makefile.am
@@ -20,7 +20,8 @@ EXTRA_DIST = \
taler-config.in \
test_helper_eddsa.conf \
test_helper_rsa.conf \
- test_helper_cs.conf
+ test_helper_cs.conf \
+ test_conversion.sh
bin_PROGRAMS = \
taler-exchange-secmod-eddsa \
@@ -80,6 +81,7 @@ libtalerutil_la_SOURCES = \
aml_signatures.c \
auditor_signatures.c \
config.c \
+ conversion.c \
crypto.c \
crypto_confirmation.c \
crypto_contract.c \
@@ -125,6 +127,7 @@ AM_TESTS_ENVIRONMENT=export TALER_PREFIX=$${TALER_PREFIX:-@libdir@};export PATH=
check_PROGRAMS = \
test_age_restriction \
test_amount \
+ test_conversion \
test_crypto \
test_helper_eddsa \
test_helper_rsa \
@@ -141,6 +144,14 @@ test_age_restriction_LDADD = \
-lgnunetutil \
libtalerutil.la
+test_conversion_SOURCES = \
+ test_conversion.c
+test_conversion_LDADD = \
+ -lgnunetjson \
+ -lgnunetutil \
+ -ljansson \
+ libtalerutil.la
+
test_amount_SOURCES = \
test_amount.c
test_amount_LDADD = \
diff --git a/src/util/conversion.c b/src/util/conversion.c
new file mode 100644
index 000000000..501ca0145
--- /dev/null
+++ b/src/util/conversion.c
@@ -0,0 +1,402 @@
+/*
+ This file is part of TALER
+ Copyright (C) 2023 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ TALER; see the file COPYING. If not, see
+*/
+/**
+ * @file conversion.c
+ * @brief helper routines to run some external JSON-to-JSON converter
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "taler_util.h"
+#include
+
+
+struct TALER_JSON_ExternalConversion
+{
+ /**
+ * Callback to call with the result.
+ */
+ TALER_JSON_JsonCallback cb;
+
+ /**
+ * Closure for @e cb.
+ */
+ void *cb_cls;
+
+ /**
+ * Handle to the helper process.
+ */
+ struct GNUNET_OS_Process *helper;
+
+ /**
+ * Pipe for the stdin of the @e helper.
+ */
+ struct GNUNET_DISK_FileHandle *chld_stdin;
+
+ /**
+ * Pipe for the stdout of the @e helper.
+ */
+ struct GNUNET_DISK_FileHandle *chld_stdout;
+
+ /**
+ * Handle to wait on the child to terminate.
+ */
+ struct GNUNET_ChildWaitHandle *cwh;
+
+ /**
+ * Task to read JSON output from the child.
+ */
+ struct GNUNET_SCHEDULER_Task *read_task;
+
+ /**
+ * Task to send JSON input to the child.
+ */
+ struct GNUNET_SCHEDULER_Task *write_task;
+
+ /**
+ * Buffer with data we need to send to the helper.
+ */
+ void *write_buf;
+
+ /**
+ * Buffer for reading data from the helper.
+ */
+ void *read_buf;
+
+ /**
+ * Total length of @e write_buf.
+ */
+ size_t write_size;
+
+ /**
+ * Current write position in @e write_buf.
+ */
+ size_t write_pos;
+
+ /**
+ * Current size of @a read_buf.
+ */
+ size_t read_size;
+
+ /**
+ * Current offset in @a read_buf.
+ */
+ size_t read_pos;
+
+};
+
+
+/**
+ * Function called when we can read more data from
+ * the child process.
+ *
+ * @param cls our `struct TALER_JSON_ExternalConversion *`
+ */
+static void
+read_cb (void *cls)
+{
+ struct TALER_JSON_ExternalConversion *ec = cls;
+
+ ec->read_task = NULL;
+ while (1)
+ {
+ ssize_t ret;
+
+ if (ec->read_size == ec->read_pos)
+ {
+ /* Grow input buffer */
+ size_t ns;
+ void *tmp;
+
+ ns = GNUNET_MAX (2 * ec->read_size,
+ 1024);
+ if (ns > GNUNET_MAX_MALLOC_CHECKED)
+ ns = GNUNET_MAX_MALLOC_CHECKED;
+ if (ec->read_size == ns)
+ {
+ /* Helper returned more than 40 MB of data! Stop reading! */
+ GNUNET_break (0);
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_file_close (ec->chld_stdin));
+ return;
+ }
+ tmp = GNUNET_malloc_large (ns);
+ if (NULL == tmp)
+ {
+ /* out of memory, also stop reading */
+ GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+ "malloc");
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_file_close (ec->chld_stdin));
+ return;
+ }
+ GNUNET_memcpy (tmp,
+ ec->read_buf,
+ ec->read_pos);
+ GNUNET_free (ec->read_buf);
+ ec->read_buf = tmp;
+ ec->read_size = ns;
+ }
+ ret = GNUNET_DISK_file_read (ec->chld_stdout,
+ ec->read_buf,
+ ec->read_size - ec->read_pos);
+ if (ret < 0)
+ {
+ if ( (EAGAIN != errno) &&
+ (EWOULDBLOCK != errno) &&
+ (EINTR != errno) )
+ {
+ GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
+ "read");
+ return;
+ }
+ break;
+ }
+ if (0 == ret)
+ {
+ /* regular end of stream, good! */
+ return;
+ }
+ GNUNET_assert (ec->read_size >= ec->read_pos + ret);
+ ec->read_pos += ret;
+ }
+ ec->read_task
+ = GNUNET_SCHEDULER_add_read_file (
+ GNUNET_TIME_UNIT_FOREVER_REL,
+ ec->chld_stdout,
+ &read_cb,
+ ec);
+}
+
+
+/**
+ * Function called when we can write more data to
+ * the child process.
+ *
+ * @param cls our `struct TALER_JSON_ExternalConversion *`
+ */
+static void
+write_cb (void *cls)
+{
+ struct TALER_JSON_ExternalConversion *ec = cls;
+ ssize_t ret;
+
+ ec->write_task = NULL;
+ while (ec->write_size > ec->write_pos)
+ {
+ ret = GNUNET_DISK_file_write (ec->chld_stdin,
+ ec->write_buf + ec->write_pos,
+ ec->write_size - ec->write_pos);
+ if (ret < 0)
+ {
+ if ( (EAGAIN != errno) &&
+ (EINTR != errno) )
+ GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
+ "write");
+ break;
+ }
+ if (0 == ret)
+ {
+ GNUNET_break (0);
+ break;
+ }
+ GNUNET_assert (ec->write_size >= ec->write_pos + ret);
+ ec->write_pos += ret;
+ }
+ if ( (ec->write_size > ec->write_pos) &&
+ ( (EAGAIN == errno) ||
+ (EWOULDBLOCK == errno) ||
+ (EINTR == errno) ) )
+ {
+ ec->write_task
+ = GNUNET_SCHEDULER_add_write_file (
+ GNUNET_TIME_UNIT_FOREVER_REL,
+ ec->chld_stdin,
+ &write_cb,
+ ec);
+ }
+ else
+ {
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_file_close (ec->chld_stdin));
+ ec->chld_stdin = NULL;
+ }
+}
+
+
+/**
+ * Defines a GNUNET_ChildCompletedCallback which is sent back
+ * upon death or completion of a child process.
+ *
+ * @param cls handle for the callback
+ * @param type type of the process
+ * @param exit_code status code of the process
+ *
+ */
+static void
+child_done_cb (void *cls,
+ enum GNUNET_OS_ProcessStatusType type,
+ long unsigned int exit_code)
+{
+ struct TALER_JSON_ExternalConversion *ec = cls;
+ json_t *j;
+ json_error_t err;
+
+ ec->cwh = NULL;
+ if (NULL != ec->read_task)
+ {
+ GNUNET_SCHEDULER_cancel (ec->read_task);
+ /* We could get the process termination notification before having drained
+ the read buffer. So drain it now, just in case. */
+ read_cb (ec);
+ }
+ if (NULL != ec->read_task)
+ {
+ GNUNET_SCHEDULER_cancel (ec->read_task);
+ ec->read_task = NULL;
+ }
+ GNUNET_OS_process_destroy (ec->helper);
+ ec->helper = NULL;
+ j = json_loadb (ec->read_buf,
+ ec->read_pos,
+ JSON_REJECT_DUPLICATES,
+ &err);
+ if (NULL == j)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ "Failed to parse JSON from helper at %d: %s\n",
+ err.position,
+ err.text);
+ }
+ ec->cb (ec->cb_cls,
+ type,
+ exit_code,
+ j);
+ json_decref (j);
+ TALER_JSON_external_conversion_stop (ec);
+}
+
+
+struct TALER_JSON_ExternalConversion *
+TALER_JSON_external_conversion_start (const json_t *input,
+ TALER_JSON_JsonCallback cb,
+ void *cb_cls,
+ const char *binary,
+ ...)
+{
+ struct TALER_JSON_ExternalConversion *ec;
+ struct GNUNET_DISK_PipeHandle *pipe_stdin;
+ struct GNUNET_DISK_PipeHandle *pipe_stdout;
+ va_list ap;
+
+ ec = GNUNET_new (struct TALER_JSON_ExternalConversion);
+ ec->cb = cb;
+ ec->cb_cls = cb_cls;
+ pipe_stdin = GNUNET_DISK_pipe (GNUNET_DISK_PF_BLOCKING_READ);
+ GNUNET_assert (NULL != pipe_stdin);
+ pipe_stdout = GNUNET_DISK_pipe (GNUNET_DISK_PF_BLOCKING_WRITE);
+ GNUNET_assert (NULL != pipe_stdout);
+ va_start (ap,
+ binary);
+ ec->helper = GNUNET_OS_start_process_va (GNUNET_OS_INHERIT_STD_ERR,
+ pipe_stdin,
+ pipe_stdout,
+ NULL,
+ binary,
+ ap);
+ va_end (ap);
+ if (NULL == ec->helper)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ "Failed to run conversion helper `%s'\n",
+ binary);
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_pipe_close (pipe_stdin));
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_pipe_close (pipe_stdout));
+ GNUNET_free (ec);
+ return NULL;
+ }
+ ec->chld_stdin =
+ GNUNET_DISK_pipe_detach_end (pipe_stdin,
+ GNUNET_DISK_PIPE_END_WRITE);
+ ec->chld_stdout =
+ GNUNET_DISK_pipe_detach_end (pipe_stdout,
+ GNUNET_DISK_PIPE_END_READ);
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_pipe_close (pipe_stdin));
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_pipe_close (pipe_stdout));
+ ec->write_buf = json_dumps (input, JSON_COMPACT);
+ ec->write_size = strlen (ec->write_buf);
+ ec->read_task
+ = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
+ ec->chld_stdout,
+ &read_cb,
+ ec);
+ ec->write_task
+ = GNUNET_SCHEDULER_add_write_file (GNUNET_TIME_UNIT_FOREVER_REL,
+ ec->chld_stdin,
+ &write_cb,
+ ec);
+ ec->cwh = GNUNET_wait_child (ec->helper,
+ &child_done_cb,
+ ec);
+ return ec;
+}
+
+
+void
+TALER_JSON_external_conversion_stop (
+ struct TALER_JSON_ExternalConversion *ec)
+{
+ if (NULL != ec->cwh)
+ {
+ GNUNET_wait_child_cancel (ec->cwh);
+ ec->cwh = NULL;
+ }
+ if (NULL != ec->helper)
+ {
+ GNUNET_break (0 ==
+ GNUNET_OS_process_kill (ec->helper,
+ SIGKILL));
+ GNUNET_OS_process_destroy (ec->helper);
+ }
+ if (NULL != ec->read_task)
+ {
+ GNUNET_SCHEDULER_cancel (ec->read_task);
+ ec->read_task = NULL;
+ }
+ if (NULL != ec->write_task)
+ {
+ GNUNET_SCHEDULER_cancel (ec->write_task);
+ ec->write_task = NULL;
+ }
+ if (NULL != ec->chld_stdin)
+ {
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_file_close (ec->chld_stdin));
+ ec->chld_stdin = NULL;
+ }
+ if (NULL != ec->chld_stdout)
+ {
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_file_close (ec->chld_stdout));
+ ec->chld_stdout = NULL;
+ }
+ GNUNET_free (ec->read_buf);
+ free (ec->write_buf);
+ GNUNET_free (ec);
+}
diff --git a/src/util/test_conversion.c b/src/util/test_conversion.c
new file mode 100644
index 000000000..00cb35e72
--- /dev/null
+++ b/src/util/test_conversion.c
@@ -0,0 +1,149 @@
+/*
+ This file is part of TALER
+ (C) 2023 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ TALER; see the file COPYING. If not, see
+*/
+/**
+ * @file util/test_conversion.c
+ * @brief Tests for conversion logic
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "taler_util.h"
+#include
+
+/**
+ * Return value from main().
+ */
+static int global_ret;
+
+/**
+ * Handle to our helper.
+ */
+static struct TALER_JSON_ExternalConversion *ec;
+
+
+/**
+ * Type of a callback that receives a JSON @a result.
+ *
+ * @param cls closure
+ * @param status_type how did the process die
+ * @apram code termination status code from the process
+ * @param result some JSON result, NULL if we failed to get an JSON output
+ */
+static void
+conv_cb (void *cls,
+ enum GNUNET_OS_ProcessStatusType status_type,
+ unsigned long code,
+ const json_t *result)
+{
+ json_t *expect;
+
+ (void) cls;
+ (void) status_type;
+ ec = NULL;
+ global_ret = 3;
+ if (42 != code)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Unexpected return value from helper: %u\n",
+ (unsigned int) code);
+ return;
+ }
+ expect = GNUNET_JSON_PACK (
+ GNUNET_JSON_pack_string ("foo",
+ "arg")
+ );
+ if (1 == json_equal (expect,
+ result))
+ {
+ global_ret = 0;
+ }
+ else
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Unexpected JSON result\n");
+ json_dumpf (result,
+ stderr,
+ JSON_INDENT (2));
+ global_ret = 4;
+ }
+ json_decref (expect);
+}
+
+
+/**
+ * Function called on shutdown/CTRL-C.
+ *
+ * @param cls NULL
+ */
+static void
+do_shutdown (void *cls)
+{
+ (void) cls;
+ if (NULL != ec)
+ {
+ GNUNET_break (0);
+ global_ret = 2;
+ TALER_JSON_external_conversion_stop (ec);
+ ec = NULL;
+ }
+}
+
+
+/**
+ * Main test function.
+ *
+ * @param cls NULL
+ */
+static void
+run (void *cls)
+{
+ json_t *input;
+
+ (void) cls;
+ GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
+ NULL);
+ input = GNUNET_JSON_PACK (
+ GNUNET_JSON_pack_string ("key",
+ "foo")
+ );
+ ec = TALER_JSON_external_conversion_start (input,
+ &conv_cb,
+ NULL,
+ "./test_conversion.sh",
+ "test_conversion.sh",
+ "arg",
+ NULL);
+ json_decref (input);
+ GNUNET_assert (NULL != ec);
+}
+
+
+int
+main (int argc,
+ const char *const argv[])
+{
+ (void) argc;
+ (void) argv;
+ unsetenv ("XDG_DATA_HOME");
+ unsetenv ("XDG_CONFIG_HOME");
+ GNUNET_log_setup ("test-conversion",
+ "WARNING",
+ NULL);
+ GNUNET_OS_init (TALER_project_data_default ());
+ global_ret = 1;
+ GNUNET_SCHEDULER_run (&run,
+ NULL);
+ return global_ret;
+}
diff --git a/src/util/test_conversion.sh b/src/util/test_conversion.sh
new file mode 100755
index 000000000..26e1a36d8
--- /dev/null
+++ b/src/util/test_conversion.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+KEY=$(jq -r .key)
+echo -n "{\"$KEY\":\"$1\"}"
+exit 42