use external helper for conversion also for KYCAID

This commit is contained in:
Christian Grothoff 2023-05-18 08:31:08 +02:00
parent b30952ed72
commit 6cc3846f4d
No known key found for this signature in database
GPG Key ID: 939E6BE1E29FC3CC
5 changed files with 422 additions and 235 deletions

View File

@ -615,3 +615,211 @@ TEH_PG_reserves_in_insert (
GNUNET_free (rrs[i].notify_s);
return qs;
}
#if 0
struct Context
{
uint64_t *reserve_uuids;
bool *transaction_duplicates;
bool *conflicts;
struct GNUNET_GenericReturnValue status;
};
/**
* Helper function to be called with the results of a SELECT statement
* that has returned @a num_results results.
*
* @param cls closure of type `struct Context *`
* @param result the postgres result
* @param num_results the number of results in @a result
*/
static void
helper_cb (void *cls,
PGresult *result,
unsigned int num_results)
{
struct Context *ctx = cls;
for (unsigned int i = 0; i<num_results; i++)
{
struct GNUNET_PQ_ResultSpec rs[] = {
GNUNET_PQ_result_spec_bool (
"transaction_duplicate",
&ctx->transaction_duplicates[i]),
GNUNET_PQ_result_spec_allow_null (
GNUNET_PQ_result_spec_uint64 ("ruuid",
&ctx->reserve_uuids[i]),
&ctx->conflicts[i]),
GNUNET_PQ_result_spec_end
};
if (GNUNET_OK !=
GNUNET_PQ_extract_result (result,
rs,
i))
{
GNUNET_break (0);
ctx->status = GNUNET_SYSERR;
return;
}
}
}
enum GNUNET_DB_QueryStatus
TEH_PG_reserves_in_insertN (
void *cls,
const struct TALER_EXCHANGEDB_ReserveInInfo *reserves,
unsigned int reserves_length,
enum GNUNET_DB_QueryStatus *results)
{
struct PostgresClosure *pg = cls;
struct TALER_PaytoHashP h_paytos[GNUNET_NZL (reserves_length)];
char *notify_s[GNUNET_NZL (reserves_length)];
struct TALER_ReservePublicKeyP *reserve_pubs[GNUNET_NZL (reserves_length)];
struct TALER_Amount *balances[GNUNET_NZL (reserves_length)];
struct GNUNET_TIME_Timestamp execution_times[GNUNET_NZL (reserves_length)];
const char *sender_account_details[GNUNET_NZL (reserves_length)];
const char *exchange_account_names[GNUNET_NZL (reserves_length)];
uint64_t wire_references[GNUNET_NZL (reserves_length)];
uint64_t reserve_uuids[GNUNET_NZL (reserves_length)];
bool transaction_duplicates[GNUNET_NZL (reserves_length)];
bool conflicts[GNUNET_NZL (reserves_length)];
struct GNUNET_TIME_Timestamp reserve_expiration
= GNUNET_TIME_relative_to_timestamp (pg->idle_reserve_expiration_time);
struct GNUNET_TIME_Timestamp gc
= GNUNET_TIME_relative_to_timestamp (pg->legal_reserve_expiration_time);
enum GNUNET_DB_QueryStatus qs;
for (unsigned int i = 0; i<reserves_length; i++)
{
const struct TALER_EXCHANGEDB_ReserveInInfo *reserve = &reserves[i];
TALER_payto_hash (reserve->sender_account_details,
&h_paytos[i]);
notify_s[i] = compute_notify_on_reserve (reserve->reserve_pub);
reserve_pubs[i] = &reserve->reserve_pub;
balances[i] = &reserve->balance;
execution_times[i] = reserve->execution_time;
sender_account_details[i] = reserve->sender_account_details;
exchange_account_names[i] = reserve->exchange_account_name;
wire_references[i] = reserve->wire_reference;
}
PREPARE (pg,
"reserves_insert_with_array",
"SELECT"
" transaction_duplicate"
",ruuid"
"FROM exchange_do_array_reserves_insert"
" ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10);");
{
struct GNUNET_PQ_QueryParam params[] = {
GNUNET_PQ_query_param_timestamp (&gc),
GNUNET_PQ_query_param_timestamp (&reserve_expiration),
GNUNET_PQ_query_param_array_auto_from_type (reserves_length,
reserve_pubs,
pg->conn),
GNUNET_PQ_query_param_array_uint64 (reserves_length,
wire_references,
pg->conn),
TALER_PQ_query_param_array_amount (reserves_length,
balances,
pg->conn),
GNUNET_PQ_query_param_array_string (reserves_length,
exchange_account_names,
pg->conn),
GNUNET_PQ_query_param_array_timestamp (reserves_length,
execution_times,
pg->conn),
GNUNET_PQ_query_param_array_bytes_same_size_cont_auto (
reserves_length,
h_paytos,
sizeof (struct GNUNET_PaytoHashP),
pg->conn),
GNUNET_PQ_query_param_array_string (reserves_length,
sender_account_details,
pg->conn),
GNUNET_PQ_query_param_array_string (reserves_length,
notify_s,
pg->conn),
GNUNET_PQ_query_param_end
};
struct Context ctx = {
.reserve_uuids = reserve_uuids,
.transaction_duplicates = transaction_duplicates,
.conflicts = conflicts,
.status = GNUNET_OK
};
qs = GNUNET_PQ_eval_prepared_multi_select (pg->conn,
"reserves_insert_with_array",
params,
&multi_res,
&ctx);
if ( (qs < 0) ||
(GNUNET_OK != ctx.status) )
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Failed to insert into reserves (%d)\n",
qs);
for (unsigned int i = 0; i<reserves_length; i++)
GNUNET_free (rrs[i].notify_s);
return qs;
}
}
for (unsigned int i = 0; i<reserves_length; i++)
{
if (! conflicts[i])
continue;
{
bool duplicate;
struct GNUNET_PQ_QueryParam params[] = {
GNUNET_PQ_query_param_auto_from_type (reserve_pubs[i]),
GNUNET_PQ_query_param_timestamp (&reserve_expiration),
GNUNET_PQ_query_param_uint64 (&wire_reference[i]),
TALER_PQ_query_param_amount (balances[i]),
GNUNET_PQ_query_param_string (exchange_account_names[i]),
GNUNET_PQ_query_param_auto_from_type (h_paytos[i]),
GNUNET_PQ_query_param_string (notify_s[i]),
GNUNET_PQ_query_param_end
};
struct GNUNET_PQ_ResultSpec rs[] = {
GNUNET_PQ_result_spec_bool ("duplicate",
&duplicate),
GNUNET_PQ_result_spec_end
};
enum GNUNET_DB_QueryStatus qs;
qs = GNUNET_PQ_eval_prepared_singleton_select (pg->conn,
"reserves_update",
params,
rs);
if (qs < 0)
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Failed to update reserves (%d)\n",
qs);
results[i] = qs;
for (unsigned int i = 0; i<reserves_length; i++)
GNUNET_free (rrs[i].notify_s);
return qs;
}
results[i] = duplicate
? GNUNET_DB_STATUS_SUCCESS_NO_RESULTS
: GNUNET_DB_STATUS_SUCCESS_ONE_RESULT;
}
}
// FIXME: convert results back for caller, too!
for (unsigned int i = 0; i<reserves_length; i++)
GNUNET_free (rrs[i].notify_s);
return qs;
}
#endif

View File

@ -20,6 +20,7 @@ EXTRA_DIST = \
persona-sample-reply.json
bin_SCRIPTS = \
taler-exchange-kyc-kycaid-converter.sh \
taler-exchange-kyc-persona-converter.sh
lib_LTLIBRARIES = \

View File

@ -12,6 +12,10 @@ PROVIDED_CHECKS = EXAMPLE_DO_NOT_USE
# How long is the KYC check valid?
KYC_KYCAID_VALIDITY = forever
# Program that converts Persona KYC data into the
# GNU Taler format.
KYC_KYCAID_CONVERTER_HELPER = taler-exchange-kyc-kycaid-converter.sh
# Authentication token to use.
KYC_KYCAID_AUTH_TOKEN = XXX

View File

@ -87,6 +87,12 @@ struct TALER_KYCLOGIC_ProviderDetails
*/
char *form_id;
/**
* Helper binary to convert attributes returned by
* KYCAID into our internal format.
*/
char *conversion_helper;
/**
* Validity time for a successful KYC process.
*/
@ -215,6 +221,12 @@ struct TALER_KYCLOGIC_WebhookHandle
*/
struct PluginState *ps;
/**
* Handle to helper process to extract attributes
* we care about.
*/
struct TALER_JSON_ExternalConversion *econ;
/**
* Our configuration details.
*/
@ -225,6 +237,11 @@ struct TALER_KYCLOGIC_WebhookHandle
*/
struct MHD_Connection *connection;
/**
* JSON response we got back, or NULL for none.
*/
json_t *json_response;
/**
* Verification ID from the service.
*/
@ -261,6 +278,11 @@ struct TALER_KYCLOGIC_WebhookHandle
*/
uint64_t process_row;
/**
* HTTP response code we got from KYCAID.
*/
unsigned int kycaid_response_code;
/**
* HTTP response code to return asynchronously.
*/
@ -277,6 +299,7 @@ static void
kycaid_unload_configuration (struct TALER_KYCLOGIC_ProviderDetails *pd)
{
curl_slist_free_all (pd->slist);
GNUNET_free (pd->conversion_helper);
GNUNET_free (pd->auth_token);
GNUNET_free (pd->form_id);
GNUNET_free (pd->section);
@ -337,6 +360,18 @@ kycaid_load_configuration (void *cls,
kycaid_unload_configuration (pd);
return NULL;
}
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_filename (ps->cfg,
provider_section_name,
"KYC_KYCAID_CONVERTER_HELPER",
&pd->conversion_helper))
{
GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
provider_section_name,
"KYC_KYCAID_CONVERTER_HELPER");
kycaid_unload_configuration (pd);
return NULL;
}
{
char *auth;
@ -695,11 +730,21 @@ kycaid_webhook_cancel (struct TALER_KYCLOGIC_WebhookHandle *wh)
GNUNET_SCHEDULER_cancel (wh->task);
wh->task = NULL;
}
if (NULL != wh->econ)
{
TALER_JSON_external_conversion_stop (wh->econ);
wh->econ = NULL;
}
if (NULL != wh->job)
{
GNUNET_CURL_job_cancel (wh->job);
wh->job = NULL;
}
if (NULL != wh->json_response)
{
json_decref (wh->json_response);
wh->json_response = NULL;
}
GNUNET_free (wh->verification_id);
GNUNET_free (wh->applicant_id);
GNUNET_free (wh->url);
@ -750,6 +795,97 @@ log_failure (json_t *verifications)
}
/**
* Type of a callback that receives a JSON @a result.
*
* @param cls closure our `struct TALER_KYCLOGIC_WebhookHandle *`
* @param status_type how did the process die
* @param code termination status code from the process
* @param result converted attribute data, NULL on failure
*/
static void
webhook_conversion_cb (void *cls,
enum GNUNET_OS_ProcessStatusType status_type,
unsigned long code,
const json_t *result)
{
struct TALER_KYCLOGIC_WebhookHandle *wh = cls;
struct GNUNET_TIME_Absolute expiration;
struct MHD_Response *resp;
wh->econ = NULL;
if ( (0 == code) ||
(NULL == result) )
{
/* No result, but *our helper* was OK => bad input */
GNUNET_break_op (0);
json_dumpf (wh->json_response,
stderr,
JSON_INDENT (2));
resp = TALER_MHD_MAKE_JSON_PACK (
GNUNET_JSON_pack_uint64 ("kycaid_http_status",
wh->kycaid_response_code),
GNUNET_JSON_pack_object_incref ("kycaid_body",
(json_t *) wh->json_response));
wh->cb (wh->cb_cls,
wh->process_row,
&wh->h_payto,
wh->pd->section,
wh->applicant_id,
wh->verification_id,
TALER_KYCLOGIC_STATUS_PROVIDER_FAILED,
GNUNET_TIME_UNIT_ZERO_ABS, /* expiration */
NULL,
MHD_HTTP_BAD_GATEWAY,
resp);
kycaid_webhook_cancel (wh);
return;
}
if (NULL == result)
{
/* Failure in our helper */
GNUNET_break (0);
json_dumpf (wh->json_response,
stderr,
JSON_INDENT (2));
resp = TALER_MHD_MAKE_JSON_PACK (
GNUNET_JSON_pack_uint64 ("kycaid_http_status",
wh->kycaid_response_code),
GNUNET_JSON_pack_object_incref ("kycaid_body",
(json_t *) wh->json_response));
wh->cb (wh->cb_cls,
wh->process_row,
&wh->h_payto,
wh->pd->section,
wh->applicant_id,
wh->verification_id,
TALER_KYCLOGIC_STATUS_PROVIDER_FAILED,
GNUNET_TIME_UNIT_ZERO_ABS, /* expiration */
NULL,
MHD_HTTP_BAD_GATEWAY,
resp);
kycaid_webhook_cancel (wh);
return;
}
expiration = GNUNET_TIME_relative_to_absolute (wh->pd->validity);
resp = MHD_create_response_from_buffer (0,
"",
MHD_RESPMEM_PERSISTENT);
wh->cb (wh->cb_cls,
wh->process_row,
&wh->h_payto,
wh->pd->section,
wh->applicant_id,
wh->verification_id,
TALER_KYCLOGIC_STATUS_SUCCESS,
expiration,
result,
MHD_HTTP_NO_CONTENT,
resp);
kycaid_webhook_cancel (wh);
}
/**
* Function called when we're done processing the
* HTTP "/applicants/{verification_id}" request.
@ -768,246 +904,20 @@ handle_webhook_finished (void *cls,
struct MHD_Response *resp;
wh->job = NULL;
wh->kycaid_response_code = response_code;
wh->json_response = json_incref ((json_t *) j);
switch (response_code)
{
case MHD_HTTP_OK:
{
const char *type;
const char *profile_status;
const char *first_name = NULL;
const char *last_name = NULL;
const char *middle_name = NULL;
const char *dob = NULL;
const char *residence_country = NULL;
const char *gender = NULL;
bool pep = false;
bool no_pep = false;
const char *company_name = NULL;
const char *business_activity_id = NULL;
const char *registration_country = NULL;
const char *email = NULL;
const char *phone = NULL;
json_t *addresses = NULL;
json_t *documents = NULL;
struct GNUNET_JSON_Specification spec[] = {
GNUNET_JSON_spec_string ("type",
&type),
GNUNET_JSON_spec_string ("profile_status",
&profile_status), /* valid, invalid, pending */
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("email",
&email),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("phone",
&phone),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_json ("addresses",
&addresses),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_json ("documents",
&documents),
NULL),
GNUNET_JSON_spec_end ()
};
struct GNUNET_JSON_Specification bspec[] = {
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("company_name",
&company_name),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("business_activity_id",
&business_activity_id),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("registration_country",
&registration_country),
NULL),
GNUNET_JSON_spec_end ()
};
struct GNUNET_JSON_Specification pspec[] = {
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("first_name",
&first_name),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("middle_name",
&middle_name),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("last_name",
&last_name),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("dob",
&dob),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("residence_country",
&residence_country),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_string ("gender",
&gender),
NULL),
GNUNET_JSON_spec_mark_optional (
GNUNET_JSON_spec_bool ("pep",
&pep),
&no_pep),
GNUNET_JSON_spec_end ()
};
struct GNUNET_JSON_Specification *ispec = NULL;
struct GNUNET_TIME_Absolute expiration;
bool no_parse;
enum TALER_KYCLOGIC_KycUserType ut;
no_parse = (GNUNET_OK !=
GNUNET_JSON_parse (j,
spec,
NULL, NULL));
if (! no_parse)
{
ut = (0 == strcasecmp ("person",
type))
? TALER_KYCLOGIC_KYC_UT_INDIVIDUAL
: TALER_KYCLOGIC_KYC_UT_BUSINESS;
ispec = (ut == TALER_KYCLOGIC_KYC_UT_INDIVIDUAL)
? pspec
: bspec;
no_parse = (GNUNET_OK !=
GNUNET_JSON_parse (j,
ispec,
NULL, NULL));
}
if (no_parse)
{
GNUNET_break_op (0);
json_dumpf (j,
stderr,
JSON_INDENT (2));
resp = TALER_MHD_MAKE_JSON_PACK (
GNUNET_JSON_pack_uint64 ("kycaid_http_status",
response_code),
GNUNET_JSON_pack_object_incref ("kycaid_body",
(json_t *) j));
wh->cb (wh->cb_cls,
wh->process_row,
&wh->h_payto,
wh->pd->section,
wh->applicant_id,
wh->verification_id,
TALER_KYCLOGIC_STATUS_PROVIDER_FAILED,
GNUNET_TIME_UNIT_ZERO_ABS, /* expiration */
NULL,
MHD_HTTP_BAD_GATEWAY,
resp);
break;
}
if (0 == strcasecmp ("valid",
profile_status = json_string_value (
json_object_get (
j,
"profile_status"));
if (0 != strcasecmp ("valid",
profile_status))
{
log_failure (json_object_get (j,
"decline_reasons"));
}
resp = MHD_create_response_from_buffer (0,
"",
MHD_RESPMEM_PERSISTENT);
if (0 == strcasecmp ("valid",
profile_status))
{
json_t *attr;
if (ut == TALER_KYCLOGIC_KYC_UT_INDIVIDUAL)
{
char *name = NULL;
if ( (NULL != last_name) ||
(NULL != first_name) ||
(NULL != middle_name) )
{
GNUNET_asprintf (&name,
"%s, %s %s",
(NULL != last_name)
? last_name
: "",
(NULL != first_name)
? first_name
: "",
(NULL != middle_name)
? middle_name
: "");
}
attr = GNUNET_JSON_PACK (
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_BIRTHDATE,
dob)),
GNUNET_JSON_pack_allow_null (
no_pep
? GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_PEP,
NULL)
: GNUNET_JSON_pack_bool (
TALER_ATTRIBUTE_PEP,
pep)),
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_FULL_NAME,
name)),
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_PHONE,
phone)),
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_EMAIL,
email)),
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_RESIDENCES,
residence_country))
);
GNUNET_free (name);
}
else
{
attr = GNUNET_JSON_PACK (
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_COMPANY_NAME,
company_name)),
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_PHONE,
phone)),
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_EMAIL,
email)),
GNUNET_JSON_pack_allow_null (
GNUNET_JSON_pack_string (
TALER_ATTRIBUTE_REGISTRATION_COUNTRY,
residence_country))
);
}
// FIXME: do something about addresses & documents!
expiration = GNUNET_TIME_relative_to_absolute (wh->pd->validity);
wh->cb (wh->cb_cls,
wh->process_row,
&wh->h_payto,
wh->pd->section,
wh->applicant_id,
wh->verification_id,
TALER_KYCLOGIC_STATUS_SUCCESS,
expiration,
attr,
MHD_HTTP_NO_CONTENT,
resp);
json_decref (attr);
}
else
{
enum TALER_KYCLOGIC_KycStatus ks;
@ -1015,6 +925,9 @@ handle_webhook_finished (void *cls,
profile_status))
? TALER_KYCLOGIC_STATUS_PENDING
: TALER_KYCLOGIC_STATUS_USER_ABORTED;
resp = MHD_create_response_from_buffer (0,
"",
MHD_RESPMEM_PERSISTENT);
wh->cb (wh->cb_cls,
wh->process_row,
&wh->h_payto,
@ -1026,9 +939,15 @@ handle_webhook_finished (void *cls,
NULL,
MHD_HTTP_NO_CONTENT,
resp);
break;
}
GNUNET_JSON_parse_free (ispec);
GNUNET_JSON_parse_free (spec);
wh->econ = TALER_JSON_external_conversion_start (j,
&webhook_conversion_cb,
wh,
wh->pd->conversion_helper,
wh->pd->conversion_helper,
NULL);
return;
}
break;
case MHD_HTTP_BAD_REQUEST:

View File

@ -0,0 +1,55 @@
#!/bin/bash
# This file is in the public domain.
#
# This code converts (some of) the JSON output from KYCAID into the GNU Taler
# specific KYC attribute data (again in JSON format). We may need to download
# and inline file data in the process, for authorization pass "-a" with the
# respective bearer token.
#
# Die if anything goes wrong.
set -eu
# Parse command-line options
while getopts ':a:' OPTION; do
case "$OPTION" in
a)
TOKEN="$OPTARG"
;;
?)
echo "Unrecognized command line option"
exit 1
;;
esac
done
# First, extract everything from stdin.
J=$(jq '{"type":.type,"email":.email,"phone":.phone,"first_name":.first_name,"name-middle":.middle_name,"last_name":.last_name,"dob":.dob,"residence_country":.residence_country,"gender":.gender,"pep":.pep,"addresses":.addresses,"documents":.documents,"company_name":.company_name,"business_activity_id":.business_activity_id,"registration_country":.registration_country}')
# TODO:
# log_failure (json_object_get (j, "decline_reasons"));
TYPE=$(echo "$J" | jq -r '.person')
if [ "person" = "${TYPE}" ]
then
# Next, combine some fields into larger values.
FULLNAME=$(echo "$J" | jq -r '[.first_name,.middle_name,.last_name]|join(" ")')
# STREET=$(echo $J | jq -r '[."street-1",."street-2"]|join(" ")')
# CITY=$(echo $J | jq -r '[.postcode,.city,."address-subdivision,.cc"]|join(" ")')
# Combine into final result for individual.
# FIXME: does jq tolerate 'pep = NULL' here?
echo "$J" | jq \
--arg full_name "${FULLNAME}" \
'{$full_name,"birthdate":.dob,"pep":.pep,"phone":."phone","email",.email,"residences":.residence_country}'
else
# Combine into final result for business.
echo "$J" | jq \
--arg full_name "${FULLNAME}" \
'{"company_name":.company_name,"phone":.phone,"email":.email,"registration_country":.registration_country}'
fi
exit 0