diff options
| author | Christian Grothoff <christian@grothoff.org> | 2023-07-25 22:17:59 +0200 | 
|---|---|---|
| committer | Christian Grothoff <christian@grothoff.org> | 2023-07-25 22:17:59 +0200 | 
| commit | eae030fc4ff5df08eb113e49eb640e7cbe2b25cb (patch) | |
| tree | 3b92e8387432ed31e431d1875ac80c86d090afdf /src/auditor | |
| parent | 28fcff7c1eb3c336278fab59c7eb352bf4b1e49a (diff) | |
-misc fixes:
Diffstat (limited to 'src/auditor')
| -rw-r--r-- | src/auditor/Makefile.am | 5 | ||||
| -rwxr-xr-x | src/auditor/generate-auditor-basedb.sh | 2 | ||||
| -rw-r--r-- | src/auditor/generate-kyc-basedb.conf | 4 | ||||
| -rwxr-xr-x | src/auditor/setup.sh | 77 | ||||
| -rwxr-xr-x | src/auditor/test-auditor.sh | 68 | ||||
| -rwxr-xr-x | src/auditor/test-kyc.sh | 784 | ||||
| -rwxr-xr-x | src/auditor/test-revocation.sh | 107 | 
7 files changed, 885 insertions, 162 deletions
| diff --git a/src/auditor/Makefile.am b/src/auditor/Makefile.am index eb8025a6..2a51ed27 100644 --- a/src/auditor/Makefile.am +++ b/src/auditor/Makefile.am @@ -207,12 +207,12 @@ taler_auditor_sync_CPPFLAGS = \  check_SCRIPTS = \    test-auditor.sh \ +  test-kyc.sh \    test-revocation.sh \    test-sync.sh  .NOTPARALLEL: -# revocation test disabled for now: need working wallet first! -TESTS = $(check_SCRIPTS) +# TESTS = $(check_SCRIPTS)  EXTRA_DIST = \    taler-auditor.in \ @@ -223,5 +223,6 @@ EXTRA_DIST = \    test-sync-out.conf \    generate-auditor-basedb.sh \    generate-auditor-basedb.conf \ +  generate-kyc-basedb.conf \    generate-revoke-basedb.sh \    $(check_SCRIPTS) diff --git a/src/auditor/generate-auditor-basedb.sh b/src/auditor/generate-auditor-basedb.sh index 1e228623..46e0813a 100755 --- a/src/auditor/generate-auditor-basedb.sh +++ b/src/auditor/generate-auditor-basedb.sh @@ -38,7 +38,7 @@ while getopts ':c:d:h' OPTION; do  done  # Where do we write the result? -if [ -z ${BASEDB:+} ] +if [ ! -v BASEDB ]  then      exit_fail "-d option required"  fi diff --git a/src/auditor/generate-kyc-basedb.conf b/src/auditor/generate-kyc-basedb.conf new file mode 100644 index 00000000..7f4a55ce --- /dev/null +++ b/src/auditor/generate-kyc-basedb.conf @@ -0,0 +1,4 @@ +# This file is in the public domain. +@INLINE@ generate-auditor-basedb.conf + +# FIXME: add options for KYC here!
\ No newline at end of file diff --git a/src/auditor/setup.sh b/src/auditor/setup.sh index 2af15494..55f0a474 100755 --- a/src/auditor/setup.sh +++ b/src/auditor/setup.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash  # This file is in the public domain  # Script to be inlined into the main test scripts. Defines function 'setup()' @@ -70,3 +70,78 @@ function get_bankaccount_transactions() {      export LIBEUFIN_SANDBOX_URL="http://localhost:18082"      libeufin-cli sandbox demobank list-transactions --bank-account $1  } + + +# Stop libeufin sandbox and nexus (if running) +function stop_libeufin() +{ +    echo -n "Stopping libeufin... " +    if [ -f "${MY_TMP_DIR:-/}/libeufin-sandbox.pid" ] +    then +        PID=$(cat "${MY_TMP_DIR}/libeufin-sandbox.pid" 2> /dev/null) +        echo "Killing libeufin sandbox $PID" +        rm "${MY_TMP_DIR}/libeufin-sandbox.pid" +        kill "$PID" 2> /dev/null || true +        wait "$PID" || true +    fi +    if [ -f "${MY_TMP_DIR:-/}/libeufin-nexus.pid" ] +    then +        PID=$(cat "${MY_TMP_DIR}/libeufin-nexus.pid" 2> /dev/null) +        echo "Killing libeufin nexus $PID" +        rm "${MY_TMP_DIR}/libeufin-nexus.pid" +        kill "$PID" 2> /dev/null || true +        wait "$PID" || true +    fi +    echo "DONE" +} + + +function launch_libeufin () { +# shellcheck disable=SC2016 +    export LIBEUFIN_SANDBOX_DB_CONNECTION='jdbc:postgresql://localhost/'"${DB}"'?socketFactory=org.newsclub.net.unix.AFUNIXSocketFactory$FactoryArg&socketFactoryArg='"$SOCKETDIR"'/.s.PGSQL.5432' +    libeufin-sandbox serve \ +                     --no-auth \ +                     --port 18082 \ +                     > "${MY_TMP_DIR}/libeufin-sandbox-stdout.log" \ +                     2> "${MY_TMP_DIR}/libeufin-sandbox-stderr.log" & +    echo $! > "${MY_TMP_DIR}/libeufin-sandbox.pid" +# shellcheck disable=SC2016 +    export LIBEUFIN_NEXUS_DB_CONNECTION='jdbc:postgresql://localhost/'"${DB}"'?socketFactory=org.newsclub.net.unix.AFUNIXSocketFactory$FactoryArg&socketFactoryArg='"$SOCKETDIR"'/.s.PGSQL.5432' +    libeufin-nexus serve \ +                   --port 8082 \ +                   2> "${MY_TMP_DIR}/libeufin-nexus-stderr.log" \ +                   > "${MY_TMP_DIR}/libeufin-nexus-stdout.log" & +    echo $! > "${MY_TMP_DIR}/libeufin-nexus.pid" +} + + + +# Downloads new transactions from the bank. +function nexus_fetch_transactions () { +    export LIBEUFIN_NEXUS_USERNAME="exchange" +    export LIBEUFIN_NEXUS_PASSWORD="x" +    export LIBEUFIN_NEXUS_URL="http://localhost:8082/" +    libeufin-cli accounts \ +                 fetch-transactions \ +                 --range-type since-last \ +                 --level report \ +                 exchange-nexus > /dev/null +    unset LIBEUFIN_NEXUS_USERNAME +    unset LIBEUFIN_NEXUS_PASSWORD +    unset LIBEUFIN_NEXUS_URL +} + + +# Instruct Nexus to all the prepared payments (= those +# POSTed to /transfer by the exchange). +function nexus_submit_to_sandbox () { +    export LIBEUFIN_NEXUS_USERNAME="exchange" +    export LIBEUFIN_NEXUS_PASSWORD="x" +    export LIBEUFIN_NEXUS_URL="http://localhost:8082/" +    libeufin-cli accounts \ +                 submit-payments\ +                 exchange-nexus +    unset LIBEUFIN_NEXUS_USERNAME +    unset LIBEUFIN_NEXUS_PASSWORD +    unset LIBEUFIN_NEXUS_URL +} diff --git a/src/auditor/test-auditor.sh b/src/auditor/test-auditor.sh index dbbe7489..9912a02a 100755 --- a/src/auditor/test-auditor.sh +++ b/src/auditor/test-auditor.sh @@ -55,28 +55,6 @@ LIBEUFIN_SETTLE_TIME=1  . setup.sh -# Stop libeufin sandbox and nexus (if running) -function stop_libeufin() -{ -    echo -n "Stopping libeufin... " -    if test -f "${MY_TMP_DIR:-/}/libeufin-sandbox.pid" -    then -        PID=$(cat "${MY_TMP_DIR}/libeufin-sandbox.pid" 2> /dev/null) -        echo "Killing libeufin sandbox $PID" -        rm "${MY_TMP_DIR}/libeufin-sandbox.pid" -        kill "$PID" 2> /dev/null || true -        wait "$PID" || true -    fi -    if test -f "${MY_TMP_DIR:-/}/libeufin-nexus.pid" -    then -        PID=$(cat "${MY_TMP_DIR}/libeufin-nexus.pid" 2> /dev/null) -        echo "Killing libeufin nexus $PID" -        rm "${MY_TMP_DIR}/libeufin-nexus.pid" -        kill "$PID" 2> /dev/null || true -        wait "$PID" || true -    fi -    echo "DONE" -}  # Cleanup exchange and libeufin between runs.  function cleanup() @@ -118,52 +96,6 @@ function exit_cleanup()  # Install cleanup handler (except for kill -9)  trap exit_cleanup EXIT -function launch_libeufin () { -# shellcheck disable=SC2016 -    export LIBEUFIN_SANDBOX_DB_CONNECTION='jdbc:postgresql://localhost/'"${DB}"'?socketFactory=org.newsclub.net.unix.AFUNIXSocketFactory$FactoryArg&socketFactoryArg='"$SOCKETDIR"'/.s.PGSQL.5432' -    export MY_TMP_DIR -    libeufin-sandbox serve --no-auth --port 18082 \ -                     > "${MY_TMP_DIR}/libeufin-sandbox-stdout.log" \ -                     2> "${MY_TMP_DIR}/libeufin-sandbox-stderr.log" & -    echo $! > "${MY_TMP_DIR}/libeufin-sandbox.pid" -# shellcheck disable=SC2016 -    export LIBEUFIN_NEXUS_DB_CONNECTION='jdbc:postgresql://localhost/'"${DB}"'?socketFactory=org.newsclub.net.unix.AFUNIXSocketFactory$FactoryArg&socketFactoryArg='"$SOCKETDIR"'/.s.PGSQL.5432' -    libeufin-nexus serve --port 8082 \ -                   2> "${MY_TMP_DIR}/libeufin-nexus-stderr.log" \ -                   > "${MY_TMP_DIR}/libeufin-nexus-stdout.log" & -    echo $! > "${MY_TMP_DIR}/libeufin-nexus.pid" -} - -# Downloads new transactions from the bank. -function nexus_fetch_transactions () { -    export LIBEUFIN_NEXUS_USERNAME="exchange" -    export LIBEUFIN_NEXUS_PASSWORD="x" -    export LIBEUFIN_NEXUS_URL="http://localhost:8082/" -    libeufin-cli accounts \ -                 fetch-transactions \ -                 --range-type since-last \ -                 --level report \ -                 exchange-nexus > /dev/null -    unset LIBEUFIN_NEXUS_USERNAME -    unset LIBEUFIN_NEXUS_PASSWORD -    unset LIBEUFIN_NEXUS_URL -} - - -# Instruct Nexus to all the prepared payments (= those -# POSTed to /transfer by the exchange). -function nexus_submit_to_sandbox () { -    export LIBEUFIN_NEXUS_USERNAME="exchange" -    export LIBEUFIN_NEXUS_PASSWORD="x" -    export LIBEUFIN_NEXUS_URL="http://localhost:8082/" -    libeufin-cli accounts \ -                 submit-payments\ -                 exchange-nexus -    unset LIBEUFIN_NEXUS_USERNAME -    unset LIBEUFIN_NEXUS_PASSWORD -    unset LIBEUFIN_NEXUS_URL -} -  # Operations to run before the actual audit  function pre_audit () { diff --git a/src/auditor/test-kyc.sh b/src/auditor/test-kyc.sh new file mode 100755 index 00000000..b984dd88 --- /dev/null +++ b/src/auditor/test-kyc.sh @@ -0,0 +1,784 @@ +#!/bin/bash +# +#  This file is part of TALER +#  Copyright (C) 2014-2023 Taler Systems SA +# +#  TALER is free software; you can redistribute it and/or modify it under the +#  terms of the GNU General Public License as published by the Free Software +#  Foundation; either version 3, or (at your option) any later version. +# +#  TALER is distributed in the hope that it will be useful, but WITHOUT ANY +#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +#  A PARTICULAR PURPOSE.  See the GNU General Public License for more details. +# +#  You should have received a copy of the GNU General Public License along with +#  TALER; see the file COPYING.  If not, If not, see <http://www.gnu.org/license> +# +# +# shellcheck disable=SC2317 +# shellcheck disable=SC1091 +# +# +# Setup database which was generated from a perfectly normal +# exchange-wallet interaction with KYC enabled and transactions +# blocked due to KYC and run the auditor against it. +# +# Check that the auditor report is as expected. +# +# Requires 'jq' tool and Postgres superuser rights! +# +set -eu +#set -x + +# Set of numbers for all the testcases. +# When adding new tests, increase the last number: +ALL_TESTS=$(seq 0 1) + +# $TESTS determines which tests we should run. +# This construction is used to make it easy to +# only run a subset of the tests. To only run a subset, +# pass the numbers of the tests to run as the FIRST +# argument to test-kyc.sh, i.e.: +# +# $ test-kyc.sh "1 3" +# +# to run tests 1 and 3 only.  By default, all tests are run. +# +TESTS=${1:-$ALL_TESTS} + +# Global variable to run the auditor processes under valgrind +# VALGRIND=valgrind +VALGRIND="" + +# Number of seconds to let libeuifn background +# tasks apply a cycle of payment submission and +# history request. +LIBEUFIN_SETTLE_TIME=1 + +. setup.sh + + +# Cleanup exchange and libeufin between runs. +function cleanup() +{ +    if test ! -z "${EPID:-}" +    then +        echo -n "Stopping exchange $EPID..." +        kill -TERM "$EPID" +        wait "$EPID" || true +        echo "DONE" +        unset EPID +    fi +    stop_libeufin +} + +# Cleanup to run whenever we exit +function exit_cleanup() +{ +    echo "Running exit-cleanup" +    if test ! -z "${POSTGRES_PATH:-}" +    then +        echo "Stopping Postgres at ${POSTGRES_PATH}" +        "${POSTGRES_PATH}/pg_ctl" \ +                        -D "$TMPDIR" \ +                        -l /dev/null \ +                        stop \ +            &> /dev/null \ +            || true +    fi +    cleanup +    for n in $(jobs -p) +    do +        kill "$n" 2> /dev/null || true +    done +    wait || true +    echo "DONE" +} + +# Install cleanup handler (except for kill -9) +trap exit_cleanup EXIT + + + +# Operations to run before the actual audit +function pre_audit () { +    # Launch bank +    echo -n "Launching bank" +    launch_libeufin +    for n in $(seq 1 80) +    do +        echo -n "." +        sleep 0.1 +        OK=1 +        wget http://localhost:18082/ \ +             -o /dev/null \ +             -O /dev/null \ +             >/dev/null \ +            && break +        OK=0 +    done +    if [ 1 != "$OK" ] +    then +        exit_skip "Failed to launch Sandbox" +    fi +    sleep "$LIBEUFIN_SETTLE_TIME" +    for n in $(seq 1 80) +    do +        echo -n "." +        sleep 0.1 +        OK=1 +        wget http://localhost:8082/ \ +             -o /dev/null \ +             -O /dev/null \ +             >/dev/null \ +            && break +        OK=0 +    done +    if [ 1 != "$OK" ] +    then +        exit_skip "Failed to launch Nexus" +    fi +    echo " DONE" +    if test "${1:-no}" = "aggregator" +    then +        echo -n "Running exchange aggregator ..." +        taler-exchange-aggregator \ +            -y \ +            -L "INFO" \ +            -t \ +            -c "$CONF" \ +            2> "${MY_TMP_DIR}/aggregator.log" \ +            || exit_fail "FAIL" +        echo " DONE" +        echo -n "Running exchange closer ..." +        taler-exchange-closer \ +            -L "INFO" \ +            -t \ +            -c "$CONF" \ +            2> "${MY_TMP_DIR}/closer.log" \ +            || exit_fail "FAIL" +        echo " DONE" +        echo -n "Running exchange transfer ..." +        taler-exchange-transfer \ +            -L "INFO" \ +            -t \ +            -c "$CONF" \ +            2> "${MY_TMP_DIR}/transfer.log" \ +            || exit_fail "FAIL" +        echo " DONE" +	    echo -n "Running Nexus payment submitter ..." +	    nexus_submit_to_sandbox +	    echo " DONE" +	    # Make outgoing transactions appear in the TWG: +	    echo -n "Download bank transactions ..." +	    nexus_fetch_transactions +	    echo " DONE" +    fi +} + +# actual audit run +function audit_only () { +    # Run the auditor! +    echo -n "Running audit(s) ..." + +    # Restart so that first run is always fresh, and second one is incremental +    taler-auditor-dbinit \ +        -r \ +        -c "$CONF" +    $VALGRIND taler-helper-auditor-aggregation \ +              -L DEBUG \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-aggregation.json \ +              2> "${MY_TMP_DIR}/test-audit-aggregation.log" \ +        || exit_fail "aggregation audit failed" +    echo -n "." +    $VALGRIND taler-helper-auditor-aggregation \ +              -L DEBUG \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-aggregation-inc.json \ +              2> "${MY_TMP_DIR}/test-audit-aggregation-inc.log" \ +        || exit_fail "incremental aggregation audit failed" +    echo -n "." +    $VALGRIND taler-helper-auditor-coins \ +              -L DEBUG \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-coins.json \ +              2> "${MY_TMP_DIR}/test-audit-coins.log" \ +        || exit_fail "coin audit failed" +    echo -n "." +    $VALGRIND taler-helper-auditor-coins \ +              -L DEBUG  \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-coins-inc.json \ +              2> "${MY_TMP_DIR}/test-audit-coins-inc.log" \ +        || exit_fail "incremental coin audit failed" +    echo -n "." +    $VALGRIND taler-helper-auditor-deposits \ +              -L DEBUG \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-deposits.json \ +              2> "${MY_TMP_DIR}/test-audit-deposits.log" \ +        || exit_fail "deposits audit failed" +    echo -n "." +    $VALGRIND taler-helper-auditor-deposits \ +              -L DEBUG \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-deposits-inc.json \ +              2> "${MY_TMP_DIR}/test-audit-deposits-inc.log" \ +        || exit_fail "incremental deposits audit failed" +    echo -n "." +    $VALGRIND taler-helper-auditor-reserves \ +              -i \ +              -L DEBUG \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-reserves.json \ +              2> "${MY_TMP_DIR}/test-audit-reserves.log" \ +        || exit_fail "reserves audit failed" +    echo -n "." +    $VALGRIND taler-helper-auditor-reserves \ +              -i \ +              -L DEBUG \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-reserves-inc.json \ +              2> "${MY_TMP_DIR}/test-audit-reserves-inc.log" \ +        || exit_fail "incremental reserves audit failed" +    echo -n "." +    rm -f "${MY_TMP_DIR}/test-wire-audit.log" +    thaw() { +        $VALGRIND taler-helper-auditor-wire \ +                  -i \ +                  -L DEBUG \ +                  -c "$CONF" \ +                  -m "$MASTER_PUB" \ +                  > test-audit-wire.json \ +                  2>> "${MY_TMP_DIR}/test-wire-audit.log" +    } +    thaw || ( echo -e " FIRST CALL TO taler-helper-auditor-wire FAILED,\nRETRY AFTER TWO SECONDS..." | tee -a "${MY_TMP_DIR}/test-wire-audit.log" +	      sleep 2 +	      thaw || exit_fail "wire audit failed" ) +    echo -n "." +    $VALGRIND taler-helper-auditor-wire \ +              -i \ +              -L DEBUG \ +              -c "$CONF" \ +              -m "$MASTER_PUB" \ +              > test-audit-wire-inc.json \ +              2> "${MY_TMP_DIR}/test-wire-audit-inc.log" \ +        || exit_fail "wire audit inc failed" +    echo -n "." + +    echo " DONE" +} + + +# Cleanup to run after the auditor +function post_audit () { +    taler-exchange-dbinit \ +        -c "$CONF" \ +        -g \ +        || exit_fail "exchange DB GC failed" + +    cleanup +    echo -n "TeXing ." +    taler-helper-auditor-render.py \ +        test-audit-aggregation.json \ +        test-audit-coins.json \ +        test-audit-deposits.json \ +        test-audit-reserves.json \ +        test-audit-wire.json \ +        < ../../contrib/auditor-report.tex.j2 \ +        > test-report.tex \ +        || exit_fail "Renderer failed" + +    echo -n "." +    timeout 10 pdflatex test-report.tex \ +            >/dev/null \ +        || exit_fail "pdflatex failed" +    echo -n "." +    timeout 10 pdflatex test-report.tex \ +            >/dev/null +    echo " DONE" +} + + +# Run audit process on current database, including report +# generation.  Pass "aggregator" as $1 to run +# $ taler-exchange-aggregator +# before auditor (to trigger pending wire transfers). +# Pass "drain" as $2 to run a drain operation as well. +function run_audit () { +    pre_audit "${1:-no}" +    if test "${2:-no}" = "drain" +    then +        echo -n "Starting exchange..." +        taler-exchange-httpd \ +            -c "${CONF}" \ +            -L INFO \ +            2> "${MY_TMP_DIR}/exchange-httpd-drain.err" & +        EPID=$! + +        # Wait for all services to be available +        for n in $(seq 1 50) +        do +            echo -n "." +            sleep 0.1 +            OK=0 +            # exchange +            wget "http://localhost:8081/seed" \ +                 -o /dev/null \ +                 -O /dev/null \ +                 >/dev/null \ +                || continue +            OK=1 +            break +        done +        echo "... DONE." +        export CONF + +        echo -n "Running taler-exchange-offline drain " + +        taler-exchange-offline \ +            -L DEBUG \ +            -c "${CONF}" \ +            drain TESTKUDOS:0.1 \ +            exchange-account-1 payto://iban/SANDBOXX/DE360679?receiver-name=Exchange+Drain \ +            upload \ +            2> "${MY_TMP_DIR}/taler-exchange-offline-drain.log" \ +            || exit_fail "offline draining failed" +        kill -TERM "$EPID" +        wait "$EPID" || true +        unset EPID +        echo -n "Running taler-exchange-drain ..." +        printf "\n" | taler-exchange-drain \ +                        -L DEBUG \ +                        -c "$CONF" \ +                        2> "${MY_TMP_DIR}/taler-exchange-drain.log" \ +            || exit_fail "FAIL" +        echo " DONE" + +        echo -n "Running taler-exchange-transfer ..." +        taler-exchange-transfer \ +            -L INFO \ +            -t \ +            -c "$CONF" \ +            2> "${MY_TMP_DIR}/drain-transfer.log" \ +            || exit_fail "FAIL" +        echo " DONE" + +        export LIBEUFIN_NEXUS_USERNAME="exchange" +        export LIBEUFIN_NEXUS_PASSWORD="x" +        export LIBEUFIN_NEXUS_URL="http://localhost:8082/" +        PAIN_UUID=$(libeufin-cli accounts list-payments exchange-nexus | jq .initiatedPayments[] | jq 'select(.submitted==false)' | jq -r .paymentInitiationId) +        if test -z "${PAIN_UUID}" +        then +            echo -n "Payment likely already submitted, running submit-payments without UUID anyway ..." +            libeufin-cli accounts \ +                         submit-payments \ +                         exchange-nexus +        else +            echo -n "Running payment submission for transaction ${PAIN_UUID} ..." +            libeufin-cli accounts \ +                         submit-payments \ +                         --payment-uuid "${PAIN_UUID}" \ +                         exchange-nexus +        fi +        echo " DONE" +        echo -n "Import outgoing transactions..." +        libeufin-cli accounts \ +                     fetch-transactions \ +                     --range-type since-last \ +                     --level report \ +                     exchange-nexus +        echo " DONE" +    fi +    audit_only +    post_audit +} + + +# Do a full reload of the (original) database +function full_reload() +{ +    echo -n "Doing full reload of the database (loading ${BASEDB}.sql into $DB at $PGHOST)... " +    dropdb "$DB" 2> /dev/null || true +    createdb -T template0 "$DB" \ +        || exit_skip "could not create database $DB (at $PGHOST)" +    # Import pre-generated database, -q(ietly) using single (-1) transaction +    psql -Aqt "$DB" \ +         -q \ +         -1 \ +         -f "${BASEDB}.sql" \ +         > /dev/null \ +        || exit_skip "Failed to load database $DB from ${BASEDB}.sql" +    echo "DONE" +    # Technically, this call shouldn't be needed as libeufin should already be stopped here... +    stop_libeufin +} + + +function test_0() { + +    echo "===========0: normal run with aggregator===========" +    run_audit aggregator +    echo "Checking output" +    # if an emergency was detected, that is a bug and we should fail +    echo -n "Test for emergencies... " +    jq -e .emergencies[0] < test-audit-coins.json > /dev/null && exit_fail "Unexpected emergency detected in ordinary run" || echo PASS +    echo -n "Test for deposit confirmation emergencies... " +    jq -e .deposit_confirmation_inconsistencies[0] < test-audit-deposits.json > /dev/null && exit_fail "Unexpected deposit confirmation inconsistency detected" || echo PASS +    echo -n "Test for emergencies by count... " +    jq -e .emergencies_by_count[0] < test-audit-coins.json > /dev/null && exit_fail "Unexpected emergency by count detected in ordinary run" || echo PASS + +    echo -n "Test for wire inconsistencies... " +    jq -e .wire_out_amount_inconsistencies[0] < test-audit-wire.json > /dev/null && exit_fail "Unexpected wire out inconsistency detected in ordinary run" +    jq -e .reserve_in_amount_inconsistencies[0] < test-audit-wire.json > /dev/null && exit_fail "Unexpected reserve in inconsistency detected in ordinary run" +    jq -e .misattribution_inconsistencies[0] < test-audit-wire.json > /dev/null && exit_fail "Unexpected misattribution inconsistency detected in ordinary run" +    jq -e .row_inconsistencies[0] < test-audit-wire.json > /dev/null && exit_fail "Unexpected row inconsistency detected in ordinary run" +    jq -e .denomination_key_validity_withdraw_inconsistencies[0] < test-audit-reserves.json > /dev/null && exit_fail "Unexpected denomination key withdraw inconsistency detected in ordinary run" +    jq -e .row_minor_inconsistencies[0] < test-audit-wire.json > /dev/null && exit_fail "Unexpected minor row inconsistency detected in ordinary run" +    jq -e .lag_details[0] < test-audit-wire.json > /dev/null && exit_fail "Unexpected lag detected in ordinary run" +    jq -e .wire_format_inconsistencies[0] < test-audit-wire.json > /dev/null && exit_fail "Unexpected wire format inconsistencies detected in ordinary run" + + +    # TODO: check operation balances are correct (once we have all transaction types and wallet is deterministic) +    # TODO: check revenue summaries are correct (once we have all transaction types and wallet is deterministic) + +    echo PASS + +    LOSS=$(jq -r .total_bad_sig_loss < test-audit-aggregation.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong total bad sig loss from aggregation, got unexpected loss of $LOSS" +    fi +    LOSS=$(jq -r .irregular_loss < test-audit-coins.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong total bad sig loss from coins, got unexpected loss of $LOSS" +    fi +    LOSS=$(jq -r .total_bad_sig_loss < test-audit-reserves.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong total bad sig loss from reserves, got unexpected loss of $LOSS" +    fi + +    echo -n "Test for wire amounts... " +    WIRED=$(jq -r .total_wire_in_delta_plus < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total wire delta plus wrong, got $WIRED" +    fi +    WIRED=$(jq -r .total_wire_in_delta_minus < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total wire delta minus wrong, got $WIRED" +    fi +    WIRED=$(jq -r .total_wire_out_delta_plus < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total wire delta plus wrong, got $WIRED" +    fi +    WIRED=$(jq -r .total_wire_out_delta_minus < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total wire delta minus wrong, got $WIRED" +    fi +    WIRED=$(jq -r .total_misattribution_in < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total misattribution in wrong, got $WIRED" +    fi +    echo "PASS" + +    echo -n "Checking for unexpected arithmetic differences " +    LOSS=$(jq -r .total_arithmetic_delta_plus < test-audit-aggregation.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong arithmetic delta from aggregations, got unexpected plus of $LOSS" +    fi +    LOSS=$(jq -r .total_arithmetic_delta_minus < test-audit-aggregation.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong arithmetic delta from aggregation, got unexpected minus of $LOSS" +    fi +    LOSS=$(jq -r .total_arithmetic_delta_plus < test-audit-coins.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong arithmetic delta from coins, got unexpected plus of $LOSS" +    fi +    LOSS=$(jq -r .total_arithmetic_delta_minus < test-audit-coins.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong arithmetic delta from coins, got unexpected minus of $LOSS" +    fi +    LOSS=$(jq -r .total_arithmetic_delta_plus < test-audit-reserves.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong arithmetic delta from reserves, got unexpected plus of $LOSS" +    fi +    LOSS=$(jq -r .total_arithmetic_delta_minus < test-audit-reserves.json) +    if [ "$LOSS" != "TESTKUDOS:0" ] +    then +        exit_fail "Wrong arithmetic delta from reserves, got unexpected minus of $LOSS" +    fi + +    jq -e .amount_arithmetic_inconsistencies[0] < test-audit-aggregation.json > /dev/null && exit_fail "Unexpected arithmetic inconsistencies from aggregations detected in ordinary run" +    jq -e .amount_arithmetic_inconsistencies[0] < test-audit-coins.json > /dev/null && exit_fail "Unexpected arithmetic inconsistencies from coins detected in ordinary run" +    jq -e .amount_arithmetic_inconsistencies[0] < test-audit-reserves.json > /dev/null && exit_fail "Unexpected arithmetic inconsistencies from reserves detected in ordinary run" +    echo "PASS" + +    echo -n "Checking for unexpected wire out differences " +    jq -e .wire_out_inconsistencies[0] < test-audit-aggregation.json > /dev/null && exit_fail "Unexpected wire out inconsistencies detected in ordinary run" +    echo "PASS" + +    # cannot easily undo aggregator, hence full reload +    full_reload + +} + + +# Run without aggregator, hence auditor should detect wire +# transfer lag! +function test_1() { + +    echo "===========1: normal run===========" +    run_audit + +    echo "Checking output" +    # if an emergency was detected, that is a bug and we should fail +    echo -n "Test for emergencies... " +    jq -e .emergencies[0] \ +       < test-audit-coins.json \ +       > /dev/null \ +        && exit_fail "Unexpected emergency detected in ordinary run"; +    echo "PASS" +    echo -n "Test for emergencies by count... " +    jq -e .emergencies_by_count[0] \ +       < test-audit-coins.json \ +       > /dev/null \ +        && exit_fail "Unexpected emergency by count detected in ordinary run" +    echo "PASS" + +    echo -n "Test for wire inconsistencies... " +    jq -e .wire_out_amount_inconsistencies[0] \ +       < test-audit-wire.json \ +       > /dev/null \ +        && exit_fail "Unexpected wire out inconsistency detected in ordinary run" +    jq -e .reserve_in_amount_inconsistencies[0] \ +       < test-audit-wire.json \ +       > /dev/null \ +        && exit_fail "Unexpected reserve in inconsistency detected in ordinary run" +    jq -e .misattribution_inconsistencies[0] \ +       < test-audit-wire.json \ +       > /dev/null \ +        && exit_fail "Unexpected misattribution inconsistency detected in ordinary run" +    jq -e .row_inconsistencies[0] \ +       < test-audit-wire.json \ +       > /dev/null \ +        && exit_fail "Unexpected row inconsistency detected in ordinary run" +    jq -e .row_minor_inconsistencies[0] \ +       < test-audit-wire.json \ +       > /dev/null \ +        && exit_fail "Unexpected minor row inconsistency detected in ordinary run" +    jq -e .wire_format_inconsistencies[0] \ +       < test-audit-wire.json \ +       > /dev/null \ +        && exit_fail "Unexpected wire format inconsistencies detected in ordinary run" + +    # TODO: check operation balances are correct (once we have all transaction types and wallet is deterministic) +    # TODO: check revenue summaries are correct (once we have all transaction types and wallet is deterministic) + +    echo "PASS" + +    echo -n "Check for lag detection... " + +    # Check wire transfer lag reported (no aggregator!) +    # NOTE: This test is EXPECTED to fail for ~1h after +    # re-generating the test database as we do not +    # report lag of less than 1h (see GRACE_PERIOD in +    # taler-helper-auditor-wire.c) +    jq -e .lag_details[0] \ +       < test-audit-wire.json \ +       > /dev/null \ +        || exit_fail "Lag not detected in run without aggregator" + +    LAG=$(jq -r .total_amount_lag < test-audit-wire.json) +    if [ "$LAG" = "TESTKUDOS:0" ] +    then +        exit_fail "Expected total lag to be non-zero" +    fi +    echo "PASS" + + +    echo -n "Test for wire amounts... " +    WIRED=$(jq -r .total_wire_in_delta_plus < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total wire delta plus wrong, got $WIRED" +    fi +    WIRED=$(jq -r .total_wire_in_delta_minus < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total wire delta minus wrong, got $WIRED" +    fi +    WIRED=$(jq -r .total_wire_out_delta_plus < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total wire delta plus wrong, got $WIRED" +    fi +    WIRED=$(jq -r .total_wire_out_delta_minus < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total wire delta minus wrong, got $WIRED" +    fi +    WIRED=$(jq -r .total_misattribution_in < test-audit-wire.json) +    if [ "$WIRED" != "TESTKUDOS:0" ] +    then +        exit_fail "Expected total misattribution in wrong, got $WIRED" +    fi +    # Database was unmodified, no need to undo +    echo "OK" +} + + + +# *************** Main test loop starts here ************** + + +# Run all the tests against the database given in $1. +# Sets $fail to 0 on success, non-zero on failure. +function check_with_database() +{ +    BASEDB="$1" +    CONF="$1.conf" +    echo "Running test suite with database $BASEDB using configuration $CONF" +    MASTER_PRIV_FILE="${BASEDB}.mpriv" +    taler-config \ +        -f \ +        -c "${CONF}" \ +        -s exchange-offline \ +        -o MASTER_PRIV_FILE \ +        -V "${MASTER_PRIV_FILE}" +    MASTER_PUB=$(gnunet-ecc -p "$MASTER_PRIV_FILE") + +    echo "MASTER PUB is ${MASTER_PUB} using file ${MASTER_PRIV_FILE}" + +    # Load database +    full_reload + +    # Run test suite +    fail=0 +    for i in $TESTS +    do +        "test_$i" +        if test 0 != $fail +        then +            break +        fi +    done +    echo "Cleanup (disabled, leaving database $DB behind)" +    # dropdb $DB +} + + + + +# *************** Main logic starts here ************** + +# ####### Setup globals ###### +# Postgres database to use (must match configuration file) +export DB="auditor-basedb" + +# test required commands exist +echo "Testing for jq" +jq -h > /dev/null || exit_skip "jq required" +echo "Testing for faketime" +faketime -h > /dev/null || exit_skip "faketime required" +# NOTE: really check for all three libeufin commands? +echo "Testing for libeufin" +libeufin-cli --help >/dev/null 2> /dev/null </dev/null || exit_skip "libeufin required" +echo "Testing for pdflatex" +which pdflatex > /dev/null </dev/null || exit_skip "pdflatex required" +echo "Testing for taler-wallet-cli" +taler-wallet-cli -h >/dev/null </dev/null 2>/dev/null || exit_skip "taler-wallet-cli required" + + +echo -n "Testing for Postgres" +# Available directly in path? +INITDB_BIN=$(command -v initdb) || true +if [[ -n "$INITDB_BIN" ]]; then +  echo " FOUND (in path) at $INITDB_BIN" +else +    HAVE_INITDB=$(find /usr -name "initdb" | head -1 2> /dev/null | grep postgres) \ +        || exit_skip " MISSING" +  echo " FOUND at $(dirname "$HAVE_INITDB")" +  INITDB_BIN=$(echo "$HAVE_INITDB" | grep bin/initdb | grep postgres | sort -n | tail -n1) +fi +POSTGRES_PATH=$(dirname "$INITDB_BIN") + +MY_TMP_DIR=$(mktemp -d /tmp/taler-auditor-basedbXXXXXX) +echo "Using $MY_TMP_DIR for logging and temporary data" +TMPDIR="$MY_TMP_DIR/postgres" +mkdir -p "$TMPDIR" +echo -n "Setting up Postgres DB at $TMPDIR ..." +$INITDB_BIN \ +    --no-sync \ +    --auth=trust \ +    -D "${TMPDIR}" \ +    > "${MY_TMP_DIR}/postgres-dbinit.log" \ +    2> "${MY_TMP_DIR}/postgres-dbinit.err" +echo "DONE" +SOCKETDIR="${TMPDIR}/sockets" +mkdir "${SOCKETDIR}" +echo -n "Launching Postgres service" +cat - >> "$TMPDIR/postgresql.conf" <<EOF +unix_socket_directories='${TMPDIR}/sockets' +fsync=off +max_wal_senders=0 +synchronous_commit=off +wal_level=minimal +listen_addresses='' +EOF +grep -v host \ +     < "$TMPDIR/pg_hba.conf" \ +     > "$TMPDIR/pg_hba.conf.new" +mv "$TMPDIR/pg_hba.conf.new" "$TMPDIR/pg_hba.conf" +"${POSTGRES_PATH}/pg_ctl" \ +                -D "$TMPDIR" \ +                -l /dev/null \ +                start \ +                > "${MY_TMP_DIR}/postgres-start.log" \ +                2> "${MY_TMP_DIR}/postgres-start.err" +echo " DONE" +PGHOST="$TMPDIR/sockets" +export PGHOST + +MYDIR="${MY_TMP_DIR}/basedb" +mkdir -p "${MYDIR}" +echo "Generating fresh database at $MYDIR" +if faketime -f '-1 d' ./generate-auditor-basedb.sh \ +            -c generate-kyc-basedb.conf \ +            -d "$MYDIR/$DB" +then +    echo -n "Reset 'auditor-basedb' database at $PGHOST ..." +    dropdb "auditor-basedb" >/dev/null 2>/dev/null || true +    createdb "auditor-basedb" || exit_skip "Could not create database '$BASEDB' at $PGHOST" +    echo " DONE" +    check_with_database "$MYDIR/$DB" +    if [ "$fail" != "0" ] +    then +        exit "$fail" +    fi +else +    echo "Generation failed" +    exit 1 +fi + +exit 0 diff --git a/src/auditor/test-revocation.sh b/src/auditor/test-revocation.sh index a1970e10..06e2b5ae 100755 --- a/src/auditor/test-revocation.sh +++ b/src/auditor/test-revocation.sh @@ -46,45 +46,12 @@ TESTS=${1:-$ALL_TESTS}  VALGRIND=""  LOGLEVEL="INFO" -# Exit, with status code "skip" (no 'real' failure) -function exit_skip() { -    echo "SKIPPING test: $1" -    exit 77 -} - -# Exit, with error message (hard failure) -function exit_fail() { -    echo "FAILING test: $1" -    exit 1 -} - -function stop_libeufin() -{ -    echo "killing libeufin..." -    if test -f "${MYDIR:-/}/libeufin-sandbox.pid" -    then -        echo "Killing libeufin sandbox" -        PID=$(cat "${MYDIR}/libeufin-sandbox.pid" 2> /dev/null) -        rm "${MYDIR}/libeufin-sandbox.pid" -        kill "$PID" 2> /dev/null || true -        wait "$PID" || true -    fi -    if test -f "${MYDIR:-/}/libeufin-nexus.pid" -    then -        echo "Killing libeufin nexus" -        PID=$(cat "${MYDIR}/libeufin-nexus.pid" 2> /dev/null) -        rm "${MYDIR}/libeufin-nexus.pid" -        kill "$PID" 2> /dev/null || true -        wait "$PID" || true -    fi -    echo "killing libeufin DONE" -} - +. setup.sh  # Cleanup to run whenever we exit  function cleanup()  { -    if test ! -z "${EPID:-}" +    if [ ! -z "${EPID:-}" ]      then          echo -n "Stopping exchange $EPID..."          kill -TERM "$EPID" @@ -99,7 +66,7 @@ function cleanup()  function exit_cleanup()  {      echo "Running exit-cleanup" -    if test ! -z "${POSTGRES_PATH:-}" +    if [ ! -z "${POSTGRES_PATH:-}" ]      then          echo "Stopping Postgres at ${POSTGRES_PATH}"          "${POSTGRES_PATH}/pg_ctl" \ @@ -121,31 +88,6 @@ function exit_cleanup()  # Install cleanup handler (except for kill -9)  trap exit_cleanup EXIT -# Downloads new transactions from the bank. -function nexus_fetch_transactions () { -    export LIBEUFIN_NEXUS_USERNAME=exchange -    export LIBEUFIN_NEXUS_PASSWORD=x -    export LIBEUFIN_NEXUS_URL=http://localhost:8082/ -    libeufin-cli accounts fetch-transactions \ -                 --range-type since-last \ -                 --level report \ -                 exchange-nexus > /dev/null -    unset LIBEUFIN_NEXUS_USERNAME -    unset LIBEUFIN_NEXUS_PASSWORD -    unset LIBEUFIN_NEXUS_URL -} - -# Instruct Nexus to all the prepared payments (= those -# POSTed to /transfer by the exchange). -function nexus_submit_to_sandbox () { -    export LIBEUFIN_NEXUS_USERNAME=exchange -    export LIBEUFIN_NEXUS_PASSWORD=x -    export LIBEUFIN_NEXUS_URL=http://localhost:8082/ -    libeufin-cli accounts submit-payments exchange-nexus -    unset LIBEUFIN_NEXUS_USERNAME -    unset LIBEUFIN_NEXUS_PASSWORD -    unset LIBEUFIN_NEXUS_URL -}  function get_payto_uri() {      export LIBEUFIN_SANDBOX_USERNAME=$1 @@ -156,21 +98,6 @@ function get_payto_uri() {          | jq --raw-output '.paytoUri'  } -function launch_libeufin () { -    export LIBEUFIN_SANDBOX_DB_CONNECTION='jdbc:postgresql://localhost/'"${DB}"'?socketFactory=org.newsclub.net.unix.AFUNIXSocketFactory$FactoryArg&socketFactoryArg='"$SOCKETDIR"'/.s.PGSQL.5432' -    libeufin-nexus serve \ -                   --port 8082 \ -                   2> "${MYDIR}/libeufin-nexus-stderr.log" \ -                   > "${MYDIR}/libeufin-nexus-stdout.log" & -    echo $! > "${MYDIR}/libeufin-nexus.pid" -    export LIBEUFIN_NEXUS_DB_CONNECTION='jdbc:postgresql://localhost/'"${DB}"'?socketFactory=org.newsclub.net.unix.AFUNIXSocketFactory$FactoryArg&socketFactoryArg='"$SOCKETDIR"'/.s.PGSQL.5432' -    libeufin-sandbox serve \ -                     --no-auth \ -                     --port 18082 \ -                     > "${MYDIR}/libeufin-sandbox-stdout.log" \ -                     2> "${MYDIR}/libeufin-sandbox-stderr.log" & -    echo $! > "${MYDIR}/libeufin-sandbox.pid" -}  # Operations to run before the actual audit  function pre_audit () { @@ -217,7 +144,7 @@ function pre_audit () {              -t \              -c "$CONF" \              -y \ -            2> "${MYDIR}/aggregator.log" \ +            2> "${MY_TMP_DIR}/aggregator.log" \              || exit_fail "FAIL"          echo " DONE"          echo -n "Running exchange closer ..." @@ -225,7 +152,7 @@ function pre_audit () {              -L "$LOGLEVEL" \              -t \              -c "$CONF" \ -            2> "${MYDIR}/closer.log" \ +            2> "${MY_TMP_DIR}/closer.log" \              || exit_fail "FAIL"          echo " DONE"          echo -n "Running exchange transfer ..." @@ -233,7 +160,7 @@ function pre_audit () {              -L "$LOGLEVEL" \              -t \              -c "$CONF" \ -            2> "${MYDIR}/transfer.log" \ +            2> "${MY_TMP_DIR}/transfer.log" \              || exit_fail "FAIL"          echo " DONE"  	    echo -n "Running Nexus payment submitter ..." @@ -789,16 +716,16 @@ else  fi  echo -n "Setting up Postgres DB"  POSTGRES_PATH=$(dirname "$INITDB_BIN") -MYDIR=$(mktemp -d /tmp/taler-auditor-basedbXXXXXX) -TMPDIR="${MYDIR}/postgres/" +MY_TMP_DIR=$(mktemp -d /tmp/taler-auditor-basedbXXXXXX) +TMPDIR="${MY_TMP_DIR}/postgres/"  mkdir -p "$TMPDIR"  echo -n "Setting up Postgres DB at $TMPDIR ..."  "$INITDB_BIN" \      --no-sync \      --auth=trust \      -D "${TMPDIR}" \ -    > "${MYDIR}/postgres-dbinit.log" \ -    2> "${MYDIR}/postgres-dbinit.err" +    > "${MY_TMP_DIR}/postgres-dbinit.log" \ +    2> "${MY_TMP_DIR}/postgres-dbinit.err"  echo " DONE"  mkdir "${TMPDIR}/sockets"  echo -n "Launching Postgres service at $POSTGRES_PATH" @@ -818,22 +745,22 @@ mv "$TMPDIR/pg_hba.conf.new" "$TMPDIR/pg_hba.conf"      -D "$TMPDIR" \      -l /dev/null \      start \ -    > "${MYDIR}/postgres-start.log" \ -    2> "${MYDIR}/postgres-start.err" +    > "${MY_TMP_DIR}/postgres-start.log" \ +    2> "${MY_TMP_DIR}/postgres-start.err"  echo " DONE"  PGHOST="$TMPDIR/sockets"  export PGHOST -echo "Generating fresh database at $MYDIR" -if faketime -f '-1 d' ./generate-revoke-basedb.sh "$MYDIR/$DB" +echo "Generating fresh database at $MY_TMP_DIR" +if faketime -f '-1 d' ./generate-revoke-basedb.sh "$MY_TMP_DIR/$DB"  then -    check_with_database "$MYDIR/$DB" +    check_with_database "$MY_TMP_DIR/$DB"      if [ "x$fail" != "x0" ]      then          exit "$fail"      else -        echo "Cleaning up $MYDIR..." -        rm -rf "$MYDIR" || echo "Removing $MYDIR failed" +        echo "Cleaning up $MY_TMP_DIR..." +        rm -rf "$MY_TMP_DIR" || echo "Removing $MY_TMP_DIR failed"      fi  else      echo "Generation failed" | 
