add helper logic for JSON2JSON conversion

src/include/taler_util.h

@@ -1,6 +1,6 @@
 /*
   This file is part of TALER
-  Copyright (C) 2014-2021 Taler Systems SA
+  Copyright (C) 2014-2023 Taler Systems SA
 
   TALER is free software; you can redistribute it and/or modify it under the
   terms of the GNU General Public License as published by the Free Software
@@ -569,6 +569,58 @@ enum GNUNET_GenericReturnValue
 TALER_JSON_parse_age_groups (const json_t *root,
                              struct TALER_AgeMask *mask);
 
+
+/**
+ * Handle to an external process that will assist
+ * with some JSON-to-JSON conversion.
+ */
+struct TALER_JSON_ExternalConversion;
+
+/**
+ * Type of a callback that receives a JSON @a result.
+ *
+ * @param cls closure
+ * @param status_type how did the process die
+ * @apram code termination status code from the process
+ * @param result some JSON result, NULL if we failed to get an JSON output
+ */
+typedef void
+(*TALER_JSON_JsonCallback) (void *cls,
+                            enum GNUNET_OS_ProcessStatusType status_type,
+                            unsigned long code,
+                            const json_t *result);
+
+
+/**
+ * Launch some external helper @a binary to convert some @a input
+ * and eventually call @a cb with the result.
+ *
+ * @param input JSON to serialize and pass to the helper process
+ * @param cb function to call on the result
+ * @param cb_cls closure for @a cb
+ * @param binary name of the binary to execute
+ * @param ... NULL-terminated list of arguments for the @a binary,
+ *        usually starting with again the name of the binary
+ * @return handle to cancel the operation (and kill the helper)
+ */
+struct TALER_JSON_ExternalConversion *
+TALER_JSON_external_conversion_start (const json_t *input,
+                                      TALER_JSON_JsonCallback cb,
+                                      void *cb_cls,
+                                      const char *binary,
+                                      ...);
+
+/**
+ * Abort external conversion, killing the process and preventing
+ * the callback from being called. Must not be called after the
+ * callback was invoked.
+ *
+ * @param[in] ec external conversion handle to cancel
+ */
+void
+TALER_JSON_external_conversion_stop (
+  struct TALER_JSON_ExternalConversion *ec);
+
 #undef __TALER_UTIL_LIB_H_INSIDE__
 
 #endif

src/util/Makefile.am

@@ -20,7 +20,8 @@ EXTRA_DIST = \
   taler-config.in \
   test_helper_eddsa.conf \
   test_helper_rsa.conf \
-  test_helper_cs.conf
+  test_helper_cs.conf \
+  test_conversion.sh
 
 bin_PROGRAMS = \
   taler-exchange-secmod-eddsa \
@@ -80,6 +81,7 @@ libtalerutil_la_SOURCES = \
   aml_signatures.c \
   auditor_signatures.c \
   config.c \
+  conversion.c \
   crypto.c \
   crypto_confirmation.c \
   crypto_contract.c \
@@ -125,6 +127,7 @@ AM_TESTS_ENVIRONMENT=export TALER_PREFIX=$${TALER_PREFIX:-@libdir@};export PATH=
 check_PROGRAMS = \
  test_age_restriction \
  test_amount \
+ test_conversion \
  test_crypto \
  test_helper_eddsa \
  test_helper_rsa \
@@ -141,6 +144,14 @@ test_age_restriction_LDADD = \
   -lgnunetutil \
   libtalerutil.la
 
+test_conversion_SOURCES = \
+  test_conversion.c
+test_conversion_LDADD = \
+  -lgnunetjson \
+  -lgnunetutil \
+  -ljansson \
+  libtalerutil.la
+
 test_amount_SOURCES = \
   test_amount.c
 test_amount_LDADD = \

src/util/conversion.c

@@ -0,0 +1,402 @@
+/*
+  This file is part of TALER
+  Copyright (C) 2023 Taler Systems SA
+
+  TALER is free software; you can redistribute it and/or modify it under the
+  terms of the GNU General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along with
+  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file conversion.c
+ * @brief helper routines to run some external JSON-to-JSON converter
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "taler_util.h"
+#include <gnunet/gnunet_util_lib.h>
+
+
+struct TALER_JSON_ExternalConversion
+{
+  /**
+   * Callback to call with the result.
+   */
+  TALER_JSON_JsonCallback cb;
+
+  /**
+   * Closure for @e cb.
+   */
+  void *cb_cls;
+
+  /**
+   * Handle to the helper process.
+   */
+  struct GNUNET_OS_Process *helper;
+
+  /**
+   * Pipe for the stdin of the @e helper.
+   */
+  struct GNUNET_DISK_FileHandle *chld_stdin;
+
+  /**
+   * Pipe for the stdout of the @e helper.
+   */
+  struct GNUNET_DISK_FileHandle *chld_stdout;
+
+  /**
+   * Handle to wait on the child to terminate.
+   */
+  struct GNUNET_ChildWaitHandle *cwh;
+
+  /**
+   * Task to read JSON output from the child.
+   */
+  struct GNUNET_SCHEDULER_Task *read_task;
+
+  /**
+   * Task to send JSON input to the child.
+   */
+  struct GNUNET_SCHEDULER_Task *write_task;
+
+  /**
+   * Buffer with data we need to send to the helper.
+   */
+  void *write_buf;
+
+  /**
+   * Buffer for reading data from the helper.
+   */
+  void *read_buf;
+
+  /**
+   * Total length of @e write_buf.
+   */
+  size_t write_size;
+
+  /**
+   * Current write position in @e write_buf.
+   */
+  size_t write_pos;
+
+  /**
+   * Current size of @a read_buf.
+   */
+  size_t read_size;
+
+  /**
+   * Current offset in @a read_buf.
+   */
+  size_t read_pos;
+
+};
+
+
+/**
+ * Function called when we can read more data from
+ * the child process.
+ *
+ * @param cls our `struct TALER_JSON_ExternalConversion *`
+ */
+static void
+read_cb (void *cls)
+{
+  struct TALER_JSON_ExternalConversion *ec = cls;
+
+  ec->read_task = NULL;
+  while (1)
+  {
+    ssize_t ret;
+
+    if (ec->read_size == ec->read_pos)
+    {
+      /* Grow input buffer */
+      size_t ns;
+      void *tmp;
+
+      ns = GNUNET_MAX (2 * ec->read_size,
+                       1024);
+      if (ns > GNUNET_MAX_MALLOC_CHECKED)
+        ns = GNUNET_MAX_MALLOC_CHECKED;
+      if (ec->read_size == ns)
+      {
+        /* Helper returned more than 40 MB of data! Stop reading! */
+        GNUNET_break (0);
+        GNUNET_break (GNUNET_OK ==
+                      GNUNET_DISK_file_close (ec->chld_stdin));
+        return;
+      }
+      tmp = GNUNET_malloc_large (ns);
+      if (NULL == tmp)
+      {
+        /* out of memory, also stop reading */
+        GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+                             "malloc");
+        GNUNET_break (GNUNET_OK ==
+                      GNUNET_DISK_file_close (ec->chld_stdin));
+        return;
+      }
+      GNUNET_memcpy (tmp,
+                     ec->read_buf,
+                     ec->read_pos);
+      GNUNET_free (ec->read_buf);
+      ec->read_buf = tmp;
+      ec->read_size = ns;
+    }
+    ret = GNUNET_DISK_file_read (ec->chld_stdout,
+                                 ec->read_buf,
+                                 ec->read_size - ec->read_pos);
+    if (ret < 0)
+    {
+      if ( (EAGAIN != errno) &&
+           (EWOULDBLOCK != errno) &&
+           (EINTR != errno) )
+      {
+        GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
+                             "read");
+        return;
+      }
+      break;
+    }
+    if (0 == ret)
+    {
+      /* regular end of stream, good! */
+      return;
+    }
+    GNUNET_assert (ec->read_size >= ec->read_pos + ret);
+    ec->read_pos += ret;
+  }
+  ec->read_task
+    = GNUNET_SCHEDULER_add_read_file (
+        GNUNET_TIME_UNIT_FOREVER_REL,
+        ec->chld_stdout,
+        &read_cb,
+        ec);
+}
+
+
+/**
+ * Function called when we can write more data to
+ * the child process.
+ *
+ * @param cls our `struct TALER_JSON_ExternalConversion *`
+ */
+static void
+write_cb (void *cls)
+{
+  struct TALER_JSON_ExternalConversion *ec = cls;
+  ssize_t ret;
+
+  ec->write_task = NULL;
+  while (ec->write_size > ec->write_pos)
+  {
+    ret = GNUNET_DISK_file_write (ec->chld_stdin,
+                                  ec->write_buf + ec->write_pos,
+                                  ec->write_size - ec->write_pos);
+    if (ret < 0)
+    {
+      if ( (EAGAIN != errno) &&
+           (EINTR != errno) )
+        GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
+                             "write");
+      break;
+    }
+    if (0 == ret)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    GNUNET_assert (ec->write_size >= ec->write_pos + ret);
+    ec->write_pos += ret;
+  }
+  if ( (ec->write_size > ec->write_pos) &&
+       ( (EAGAIN == errno) ||
+         (EWOULDBLOCK == errno) ||
+         (EINTR == errno) ) )
+  {
+    ec->write_task
+      = GNUNET_SCHEDULER_add_write_file (
+          GNUNET_TIME_UNIT_FOREVER_REL,
+          ec->chld_stdin,
+          &write_cb,
+          ec);
+  }
+  else
+  {
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_DISK_file_close (ec->chld_stdin));
+    ec->chld_stdin = NULL;
+  }
+}
+
+
+/**
+ * Defines a GNUNET_ChildCompletedCallback which is sent back
+ * upon death or completion of a child process.
+ *
+ * @param cls handle for the callback
+ * @param type type of the process
+ * @param exit_code status code of the process
+ *
+ */
+static void
+child_done_cb (void *cls,
+               enum GNUNET_OS_ProcessStatusType type,
+               long unsigned int exit_code)
+{
+  struct TALER_JSON_ExternalConversion *ec = cls;
+  json_t *j;
+  json_error_t err;
+
+  ec->cwh = NULL;
+  if (NULL != ec->read_task)
+  {
+    GNUNET_SCHEDULER_cancel (ec->read_task);
+    /* We could get the process termination notification before having drained
+       the read buffer. So drain it now, just in case. */
+    read_cb (ec);
+  }
+  if (NULL != ec->read_task)
+  {
+    GNUNET_SCHEDULER_cancel (ec->read_task);
+    ec->read_task = NULL;
+  }
+  GNUNET_OS_process_destroy (ec->helper);
+  ec->helper = NULL;
+  j = json_loadb (ec->read_buf,
+                  ec->read_pos,
+                  JSON_REJECT_DUPLICATES,
+                  &err);
+  if (NULL == j)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Failed to parse JSON from helper at %d: %s\n",
+                err.position,
+                err.text);
+  }
+  ec->cb (ec->cb_cls,
+          type,
+          exit_code,
+          j);
+  json_decref (j);
+  TALER_JSON_external_conversion_stop (ec);
+}
+
+
+struct TALER_JSON_ExternalConversion *
+TALER_JSON_external_conversion_start (const json_t *input,
+                                      TALER_JSON_JsonCallback cb,
+                                      void *cb_cls,
+                                      const char *binary,
+                                      ...)
+{
+  struct TALER_JSON_ExternalConversion *ec;
+  struct GNUNET_DISK_PipeHandle *pipe_stdin;
+  struct GNUNET_DISK_PipeHandle *pipe_stdout;
+  va_list ap;
+
+  ec = GNUNET_new (struct TALER_JSON_ExternalConversion);
+  ec->cb = cb;
+  ec->cb_cls = cb_cls;
+  pipe_stdin = GNUNET_DISK_pipe (GNUNET_DISK_PF_BLOCKING_READ);
+  GNUNET_assert (NULL != pipe_stdin);
+  pipe_stdout = GNUNET_DISK_pipe (GNUNET_DISK_PF_BLOCKING_WRITE);
+  GNUNET_assert (NULL != pipe_stdout);
+  va_start (ap,
+            binary);
+  ec->helper = GNUNET_OS_start_process_va (GNUNET_OS_INHERIT_STD_ERR,
+                                           pipe_stdin,
+                                           pipe_stdout,
+                                           NULL,
+                                           binary,
+                                           ap);
+  va_end (ap);
+  if (NULL == ec->helper)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Failed to run conversion helper `%s'\n",
+                binary);
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_DISK_pipe_close (pipe_stdin));
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_DISK_pipe_close (pipe_stdout));
+    GNUNET_free (ec);
+    return NULL;
+  }
+  ec->chld_stdin =
+    GNUNET_DISK_pipe_detach_end (pipe_stdin,
+                                 GNUNET_DISK_PIPE_END_WRITE);
+  ec->chld_stdout =
+    GNUNET_DISK_pipe_detach_end (pipe_stdout,
+                                 GNUNET_DISK_PIPE_END_READ);
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_DISK_pipe_close (pipe_stdin));
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_DISK_pipe_close (pipe_stdout));
+  ec->write_buf = json_dumps (input, JSON_COMPACT);
+  ec->write_size = strlen (ec->write_buf);
+  ec->read_task
+    = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
+                                      ec->chld_stdout,
+                                      &read_cb,
+                                      ec);
+  ec->write_task
+    = GNUNET_SCHEDULER_add_write_file (GNUNET_TIME_UNIT_FOREVER_REL,
+                                       ec->chld_stdin,
+                                       &write_cb,
+                                       ec);
+  ec->cwh = GNUNET_wait_child (ec->helper,
+                               &child_done_cb,
+                               ec);
+  return ec;
+}
+
+
+void
+TALER_JSON_external_conversion_stop (
+  struct TALER_JSON_ExternalConversion *ec)
+{
+  if (NULL != ec->cwh)
+  {
+    GNUNET_wait_child_cancel (ec->cwh);
+    ec->cwh = NULL;
+  }
+  if (NULL != ec->helper)
+  {
+    GNUNET_break (0 ==
+                  GNUNET_OS_process_kill (ec->helper,
+                                          SIGKILL));
+    GNUNET_OS_process_destroy (ec->helper);
+  }
+  if (NULL != ec->read_task)
+  {
+    GNUNET_SCHEDULER_cancel (ec->read_task);
+    ec->read_task = NULL;
+  }
+  if (NULL != ec->write_task)
+  {
+    GNUNET_SCHEDULER_cancel (ec->write_task);
+    ec->write_task = NULL;
+  }
+  if (NULL != ec->chld_stdin)
+  {
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_DISK_file_close (ec->chld_stdin));
+    ec->chld_stdin = NULL;
+  }
+  if (NULL != ec->chld_stdout)
+  {
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_DISK_file_close (ec->chld_stdout));
+    ec->chld_stdout = NULL;
+  }
+  GNUNET_free (ec->read_buf);
+  free (ec->write_buf);
+  GNUNET_free (ec);
+}

src/util/test_conversion.c

@@ -0,0 +1,149 @@
+/*
+  This file is part of TALER
+  (C) 2023 Taler Systems SA
+
+  TALER is free software; you can redistribute it and/or modify it under the
+  terms of the GNU General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along with
+  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file util/test_conversion.c
+ * @brief Tests for conversion logic
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "taler_util.h"
+#include <gnunet/gnunet_json_lib.h>
+
+/**
+ * Return value from main().
+ */
+static int global_ret;
+
+/**
+ * Handle to our helper.
+ */
+static struct TALER_JSON_ExternalConversion *ec;
+
+
+/**
+ * Type of a callback that receives a JSON @a result.
+ *
+ * @param cls closure
+ * @param status_type how did the process die
+ * @apram code termination status code from the process
+ * @param result some JSON result, NULL if we failed to get an JSON output
+ */
+static void
+conv_cb (void *cls,
+         enum GNUNET_OS_ProcessStatusType status_type,
+         unsigned long code,
+         const json_t *result)
+{
+  json_t *expect;
+
+  (void) cls;
+  (void) status_type;
+  ec = NULL;
+  global_ret = 3;
+  if (42 != code)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Unexpected return value from helper: %u\n",
+                (unsigned int) code);
+    return;
+  }
+  expect = GNUNET_JSON_PACK (
+    GNUNET_JSON_pack_string ("foo",
+                             "arg")
+    );
+  if (1 == json_equal (expect,
+                       result))
+  {
+    global_ret = 0;
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Unexpected JSON result\n");
+    json_dumpf (result,
+                stderr,
+                JSON_INDENT (2));
+    global_ret = 4;
+  }
+  json_decref (expect);
+}
+
+
+/**
+ * Function called on shutdown/CTRL-C.
+ *
+ * @param cls NULL
+ */
+static void
+do_shutdown (void *cls)
+{
+  (void) cls;
+  if (NULL != ec)
+  {
+    GNUNET_break (0);
+    global_ret = 2;
+    TALER_JSON_external_conversion_stop (ec);
+    ec = NULL;
+  }
+}
+
+
+/**
+ * Main test function.
+ *
+ * @param cls NULL
+ */
+static void
+run (void *cls)
+{
+  json_t *input;
+
+  (void) cls;
+  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
+                                 NULL);
+  input = GNUNET_JSON_PACK (
+    GNUNET_JSON_pack_string ("key",
+                             "foo")
+    );
+  ec = TALER_JSON_external_conversion_start (input,
+                                             &conv_cb,
+                                             NULL,
+                                             "./test_conversion.sh",
+                                             "test_conversion.sh",
+                                             "arg",
+                                             NULL);
+  json_decref (input);
+  GNUNET_assert (NULL != ec);
+}
+
+
+int
+main (int argc,
+      const char *const argv[])
+{
+  (void) argc;
+  (void) argv;
+  unsetenv ("XDG_DATA_HOME");
+  unsetenv ("XDG_CONFIG_HOME");
+  GNUNET_log_setup ("test-conversion",
+                    "WARNING",
+                    NULL);
+  GNUNET_OS_init (TALER_project_data_default ());
+  global_ret = 1;
+  GNUNET_SCHEDULER_run (&run,
+                        NULL);
+  return global_ret;
+}

src/util/test_conversion.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+KEY=$(jq -r .key)
+echo -n "{\"$KEY\":\"$1\"}"
+exit 42