harness: reusable test env

packages/taler-harness/src/harness/harness.ts

@@ -489,6 +489,7 @@ export interface BankConfig {
   database: string;
   allowRegistrations: boolean;
   maxDebt?: string;
+  overrideTestDir?: string;
 }
 
 export interface FakeBankConfig {
@@ -534,6 +535,14 @@ function setCoin(config: Configuration, c: CoinConfig) {
   }
 }
 
+function backoffStart(): number {
+  return 10;
+}
+
+function backoffIncrement(n: number): number {
+  return Math.max(n * 2, 1000);
+}
+
 /**
  * Send an HTTP request until it succeeds or the process dies.
  */
@@ -545,6 +554,7 @@ export async function pingProc(
   if (!proc || proc.proc.exitCode !== null) {
     throw Error(`service process ${serviceName} not started, can't ping`);
   }
+  let nextDelay = backoffStart();
   while (true) {
     try {
       logger.trace(`pinging ${serviceName} at ${url}`);
@@ -554,7 +564,8 @@ export async function pingProc(
     } catch (e: any) {
       logger.warn(`service ${serviceName} not ready:`, e.toString());
       //console.log(e);
-      await delayMs(1000);
+      await delayMs(nextDelay);
+      nextDelay = backoffIncrement(nextDelay);
     }
     if (!proc || proc.proc.exitCode != null || proc.proc.signalCode != null) {
       throw Error(`service process ${serviceName} stopped unexpectedly`);
@@ -885,19 +896,38 @@ export class FakebankService
     bc: BankConfig,
   ): Promise<FakebankService> {
     const config = new Configuration();
-    setTalerPaths(config, gc.testDir + "/talerhome");
+    const testDir = bc.overrideTestDir ?? gc.testDir;
+    setTalerPaths(config, testDir + "/talerhome");
     config.setString("taler", "currency", bc.currency);
     config.setString("bank", "http_port", `${bc.httpPort}`);
     config.setString("bank", "serve", "http");
     config.setString("bank", "max_debt_bank", `${bc.currency}:999999`);
     config.setString("bank", "max_debt", bc.maxDebt ?? `${bc.currency}:100`);
     config.setString("bank", "ram_limit", `${1024}`);
-    const cfgFilename = gc.testDir + "/bank.conf";
+    const cfgFilename = testDir + "/bank.conf";
     config.write(cfgFilename);
 
     return new FakebankService(gc, bc, cfgFilename);
   }
 
+  static fromExistingConfig(
+    gc: GlobalTestState,
+    opts: { overridePath?: string },
+  ): FakebankService {
+    const testDir = opts.overridePath ?? gc.testDir;
+    const cfgFilename = testDir + `/bank.conf`;
+    const config = Configuration.load(cfgFilename);
+    const bc: BankConfig = {
+      allowRegistrations:
+        config.getYesNo("bank", "allow_registrations").orUndefined() ?? true,
+      currency: config.getString("taler", "currency").required(),
+      database: "none",
+      httpPort: config.getNumber("bank", "http_port").required(),
+      maxDebt: config.getString("bank", "max_debt").required(),
+    };
+    return new FakebankService(gc, bc, cfgFilename);
+  }
+
   setSuggestedExchange(e: ExchangeServiceInterface, exchangePayto: string) {
     if (!!this.proc) {
       throw Error("Can't set suggested exchange while bank is running.");
@@ -981,6 +1011,7 @@ export interface ExchangeConfig {
   roundUnit?: string;
   httpPort: number;
   database: string;
+  overrideTestDir?: string;
 }
 
 export interface ExchangeServiceInterface {
@@ -991,8 +1022,13 @@ export interface ExchangeServiceInterface {
 }
 
 export class ExchangeService implements ExchangeServiceInterface {
-  static fromExistingConfig(gc: GlobalTestState, exchangeName: string) {
+  static fromExistingConfig(
-    const cfgFilename = gc.testDir + `/exchange-${exchangeName}.conf`;
+    gc: GlobalTestState,
+    exchangeName: string,
+    opts: { overridePath?: string },
+  ) {
+    const testDir = opts.overridePath ?? gc.testDir;
+    const cfgFilename = testDir + `/exchange-${exchangeName}.conf`;
     const config = Configuration.load(cfgFilename);
     const ec: ExchangeConfig = {
       currency: config.getString("taler", "currency").required(),
@@ -1103,7 +1139,9 @@ export class ExchangeService implements ExchangeServiceInterface {
   }
 
   static create(gc: GlobalTestState, e: ExchangeConfig) {
+    const testDir = e.overrideTestDir ?? gc.testDir;
     const config = new Configuration();
+    setTalerPaths(config, testDir + "/talerhome");
     config.setString("taler", "currency", e.currency);
     // Required by the exchange but not really used yet.
     config.setString("exchange", "aml_threshold", `${e.currency}:1000000`);
@@ -1112,7 +1150,6 @@ export class ExchangeService implements ExchangeServiceInterface {
       "currency_round_unit",
       e.roundUnit ?? `${e.currency}:0.01`,
     );
-    setTalerPaths(config, gc.testDir + "/talerhome");
     config.setString(
       "exchange",
       "revocation_dir",
@@ -1149,7 +1186,7 @@ export class ExchangeService implements ExchangeServiceInterface {
 
     fs.writeFileSync(masterPrivFile, Buffer.from(exchangeMasterKey.eddsaPriv));
 
-    const cfgFilename = gc.testDir + `/exchange-${e.name}.conf`;
+    const cfgFilename = testDir + `/exchange-${e.name}.conf`;
     config.write(cfgFilename);
     return new ExchangeService(gc, e, cfgFilename, exchangeMasterKey);
   }
@@ -1553,6 +1590,7 @@ export interface MerchantConfig {
   currency: string;
   httpPort: number;
   database: string;
+  overrideTestDir?: string;
 }
 
 export interface PrivateOrderStatusQuery {
@@ -1798,8 +1836,13 @@ export interface CreateMerchantTippingReserveRequest {
 }
 
 export class MerchantService implements MerchantServiceInterface {
-  static fromExistingConfig(gc: GlobalTestState, name: string) {
+  static fromExistingConfig(
-    const cfgFilename = gc.testDir + `/merchant-${name}.conf`;
+    gc: GlobalTestState,
+    name: string,
+    opts: { overridePath?: string },
+  ) {
+    const testDir = opts.overridePath ?? gc.testDir;
+    const cfgFilename = testDir + `/merchant-${name}.conf`;
     const config = Configuration.load(cfgFilename);
     const mc: MerchantConfig = {
       currency: config.getString("taler", "currency").required(),
@@ -1894,11 +1937,12 @@ export class MerchantService implements MerchantServiceInterface {
     gc: GlobalTestState,
     mc: MerchantConfig,
   ): Promise<MerchantService> {
+    const testDir = mc.overrideTestDir ?? gc.testDir;
     const config = new Configuration();
     config.setString("taler", "currency", mc.currency);
 
-    const cfgFilename = gc.testDir + `/merchant-${mc.name}.conf`;
+    const cfgFilename = testDir + `/merchant-${mc.name}.conf`;
-    setTalerPaths(config, gc.testDir + "/talerhome");
+    setTalerPaths(config, testDir + "/talerhome");
     config.setString("merchant", "serve", "tcp");
     config.setString("merchant", "port", `${mc.httpPort}`);
     config.setString(

packages/taler-harness/src/harness/helpers.ts

@@ -32,6 +32,7 @@ import {
   NotificationType,
   WalletNotification,
   TransactionMajorState,
+  Logger,
 } from "@gnu-taler/taler-util";
 import {
   BankAccessApi,
@@ -49,6 +50,7 @@ import {
   DbInfo,
   ExchangeService,
   ExchangeServiceInterface,
+  FakebankService,
   getPayto,
   GlobalTestState,
   MerchantPrivateApi,
@@ -62,6 +64,10 @@ import {
   WithAuthorization,
 } from "./harness.js";
 
+import * as fs from "fs";
+
+const logger = new Logger("helpers.ts");
+
 /**
  * @deprecated
  */
@@ -212,48 +218,103 @@ export async function createSimpleTestkudosEnvironment(
 export async function useSharedTestkudosEnvironment(t: GlobalTestState) {
   const coinConfig: CoinConfig[] = defaultCoinConfig.map((x) => x("TESTKUDOS"));
 
+  // FIXME: We should probably have some file to indicate that
+  // the previous env setup finished successfully.
+
+  const sharedDir = `/tmp/taler-harness@${process.env.USER}`;
+
+  fs.mkdirSync(sharedDir, { recursive: true });
+
   const db = await setupSharedDb(t);
 
-  const bank = await BankService.create(t, {
+  let bank: FakebankService;
-    allowRegistrations: true,
-    currency: "TESTKUDOS",
-    database: db.connStr,
-    httpPort: 8082,
-  });
 
-  const exchange = ExchangeService.create(t, {
+  const prevSetupDone = fs.existsSync(sharedDir + "/setup-done");
-    name: "testexchange-1",
-    currency: "TESTKUDOS",
-    httpPort: 8081,
-    database: db.connStr,
-  });
 
-  const merchant = await MerchantService.create(t, {
+  logger.info(`previous setup done: ${prevSetupDone}`);
-    name: "testmerchant-1",
+
-    currency: "TESTKUDOS",
+  if (fs.existsSync(sharedDir + "/bank.conf")) {
-    httpPort: 8083,
+    logger.info("reusing existing bank");
-    database: db.connStr,
+    bank = BankService.fromExistingConfig(t, {
-  });
+      overridePath: sharedDir,
+    });
+  } else {
+    logger.info("creating new bank config");
+    bank = await BankService.create(t, {
+      allowRegistrations: true,
+      currency: "TESTKUDOS",
+      database: db.connStr,
+      httpPort: 8082,
+      overrideTestDir: sharedDir,
+    });
+  }
+
+  logger.info("setting up exchange");
+
+  const exchangeName = "testexchange-1";
+  const exchangeConfigFilename = sharedDir + `/exchange-${exchangeName}}`;
+
+  let exchange: ExchangeService;
+
+  if (fs.existsSync(exchangeConfigFilename)) {
+    exchange = ExchangeService.fromExistingConfig(t, exchangeName, {
+      overridePath: sharedDir,
+    });
+  } else {
+    exchange = ExchangeService.create(t, {
+      name: "testexchange-1",
+      currency: "TESTKUDOS",
+      httpPort: 8081,
+      database: db.connStr,
+      overrideTestDir: sharedDir,
+    });
+  }
+
+  logger.info("setting up exchange");
+
+  let merchant: MerchantService;
+  const merchantName = "testmerchant-1";
+  const merchantConfigFilename = sharedDir + `/merchant-${merchantName}}`;
+
+  if (fs.existsSync(merchantConfigFilename)) {
+    merchant = MerchantService.fromExistingConfig(t, merchantName, {
+      overridePath: sharedDir,
+    });
+  } else {
+    merchant = await MerchantService.create(t, {
+      name: "testmerchant-1",
+      currency: "TESTKUDOS",
+      httpPort: 8083,
+      database: db.connStr,
+      overrideTestDir: sharedDir,
+    });
+  }
+
+  logger.info("creating bank account for exchange");
 
   const exchangeBankAccount = await bank.createExchangeAccount(
     "myexchange",
     "x",
   );
+
+  logger.info("creating exchange bank account");
   await exchange.addBankAccount("1", exchangeBankAccount);
 
   bank.setSuggestedExchange(exchange, exchangeBankAccount.accountPaytoUri);
 
+  exchange.addCoinConfigList(coinConfig);
+
+  merchant.addExchange(exchange);
+
+  logger.info("basic setup done, starting services");
+
   await bank.start();
 
   await bank.pingUntilAvailable();
 
-  exchange.addCoinConfigList(coinConfig);
-
   await exchange.start();
   await exchange.pingUntilAvailable();
 
-  merchant.addExchange(exchange);
-
   await merchant.start();
   await merchant.pingUntilAvailable();
 
@@ -282,6 +343,8 @@ export async function useSharedTestkudosEnvironment(t: GlobalTestState) {
 
   console.log("setup done!");
 
+  fs.writeFileSync(sharedDir + "/setup-done", "OK");
+
   return {
     commonDb: db,
     exchange,

packages/taler-harness/src/integrationtests/testrunner.ts

@@ -14,7 +14,7 @@
  GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
  */
 
-import { CancellationToken, Logger, minimatch } from "@gnu-taler/taler-util";
+import { CancellationToken, Logger, minimatch, setGlobalLogLevelFromString } from "@gnu-taler/taler-util";
 import * as child_process from "child_process";
 import * as fs from "fs";
 import * as os from "os";
@@ -494,6 +494,8 @@ if (runTestInstrStr && process.argv.includes("__TWCLI_TESTWORKER")) {
     runTestInstrStr,
   ) as RunTestChildInstruction;
 
+  setGlobalLogLevelFromString("TRACE");
+
   process.on("disconnect", () => {
     logger.trace("got disconnect from parent");
     process.exit(3);

packages/taler-util/src/logging.ts

@@ -32,13 +32,13 @@ export enum LogLevel {
   None = "none",
 }
 
-export let globalLogLevel = LogLevel.Info;
+let globalLogLevel = LogLevel.Info;
+const byTagLogLevel: Record<string, LogLevel> = {};
 
 export function setGlobalLogLevelFromString(logLevelStr: string): void {
   globalLogLevel = getLevelForString(logLevelStr);
 }
 
-export const byTagLogLevel: Record<string, LogLevel> = {};
 export function setLogLevelFromString(tag: string, logLevelStr: string): void {
   byTagLogLevel[tag] = getLevelForString(logLevelStr);
 }