Merge branch 'master' into age-withdraw

1 files changed, 142 insertions, 46 deletions

diff --git a/packages/taler-harness/src/harness/harness.ts b/packages/taler-harness/src/harness/harness.ts
index c9202c60e..926a0c93b 100644
--- a/packages/taler-harness/src/harness/harness.ts
+++ b/packages/taler-harness/src/harness/harness.ts
@@ -467,12 +467,29 @@ export async function setupDb(t: GlobalTestState): Promise<DbInfo> {
   };
 }
 
+/**
+ * Make sure that the taler-integrationtest-shared database exists.
+ * Don't delete it if it already exists.
+ */
+export async function setupSharedDb(t: GlobalTestState): Promise<DbInfo> {
+  const dbname = "taler-integrationtest-shared";
+  const databases = await runCommand(t, "list-dbs", "psql", ["-Aqtl"]);
+  if (databases.indexOf("taler-integrationtest-shared") < 0) {
+    await runCommand(t, "createdb", "createdb", [dbname]);
+  }
+  return {
+    connStr: `postgres:///${dbname}`,
+    dbname,
+  };
+}
+
 export interface BankConfig {
   currency: string;
   httpPort: number;
   database: string;
   allowRegistrations: boolean;
   maxDebt?: string;
+  overrideTestDir?: string;
 }
 
 export interface FakeBankConfig {
@@ -518,6 +535,14 @@ function setCoin(config: Configuration, c: CoinConfig) {
   }
 }
 
+function backoffStart(): number {
+  return 10;
+}
+
+function backoffIncrement(n: number): number {
+  return Math.min(Math.floor(n * 1.5), 1000);
+}
+
 /**
  * Send an HTTP request until it succeeds or the process dies.
  */
@@ -529,6 +554,7 @@ export async function pingProc(
   if (!proc || proc.proc.exitCode !== null) {
     throw Error(`service process ${serviceName} not started, can't ping`);
   }
+  let nextDelay = backoffStart();
   while (true) {
     try {
       logger.trace(`pinging ${serviceName} at ${url}`);
@@ -537,8 +563,9 @@ export async function pingProc(
       return;
     } catch (e: any) {
       logger.warn(`service ${serviceName} not ready:`, e.toString());
-      //console.log(e);
-      await delayMs(1000);
+      logger.info(`waiting ${nextDelay}ms on ${serviceName}`);
+      await delayMs(nextDelay);
+      nextDelay = backoffIncrement(nextDelay);
     }
     if (!proc || proc.proc.exitCode != null || proc.proc.signalCode != null) {
       throw Error(`service process ${serviceName} stopped unexpectedly`);
@@ -857,31 +884,57 @@ export class FakebankService
     accountPassword: string;
   }[] = [];
 
+  /**
+   * Create a new fakebank service handle.
+   *
+   * First generates the configuration for the fakebank and
+   * then creates a fakebank handle, but doesn't start the fakebank
+   * service yet.
+   */
   static async create(
     gc: GlobalTestState,
     bc: BankConfig,
   ): Promise<FakebankService> {
     const config = new Configuration();
-    setTalerPaths(config, gc.testDir + "/talerhome");
+    const testDir = bc.overrideTestDir ?? gc.testDir;
+    setTalerPaths(config, testDir + "/talerhome");
     config.setString("taler", "currency", bc.currency);
     config.setString("bank", "http_port", `${bc.httpPort}`);
     config.setString("bank", "serve", "http");
     config.setString("bank", "max_debt_bank", `${bc.currency}:999999`);
     config.setString("bank", "max_debt", bc.maxDebt ?? `${bc.currency}:100`);
     config.setString("bank", "ram_limit", `${1024}`);
-    const cfgFilename = gc.testDir + "/bank.conf";
-    config.write(cfgFilename);
+    const cfgFilename = testDir + "/bank.conf";
+    config.write(cfgFilename, { excludeDefaults: true });
 
     return new FakebankService(gc, bc, cfgFilename);
   }
 
+  static fromExistingConfig(
+    gc: GlobalTestState,
+    opts: { overridePath?: string },
+  ): FakebankService {
+    const testDir = opts.overridePath ?? gc.testDir;
+    const cfgFilename = testDir + `/bank.conf`;
+    const config = Configuration.load(cfgFilename);
+    const bc: BankConfig = {
+      allowRegistrations:
+        config.getYesNo("bank", "allow_registrations").orUndefined() ?? true,
+      currency: config.getString("taler", "currency").required(),
+      database: "none",
+      httpPort: config.getNumber("bank", "http_port").required(),
+      maxDebt: config.getString("bank", "max_debt").required(),
+    };
+    return new FakebankService(gc, bc, cfgFilename);
+  }
+
   setSuggestedExchange(e: ExchangeServiceInterface, exchangePayto: string) {
     if (!!this.proc) {
       throw Error("Can't set suggested exchange while bank is running.");
     }
     const config = Configuration.load(this.configFile);
     config.setString("bank", "suggested_exchange", e.baseUrl);
-    config.write(this.configFile);
+    config.write(this.configFile, { excludeDefaults: true });
   }
 
   get baseUrl(): string {
@@ -958,6 +1011,7 @@ export interface ExchangeConfig {
   roundUnit?: string;
   httpPort: number;
   database: string;
+  overrideTestDir?: string;
 }
 
 export interface ExchangeServiceInterface {
@@ -968,8 +1022,13 @@ export interface ExchangeServiceInterface {
 }
 
 export class ExchangeService implements ExchangeServiceInterface {
-  static fromExistingConfig(gc: GlobalTestState, exchangeName: string) {
-    const cfgFilename = gc.testDir + `/exchange-${exchangeName}.conf`;
+  static fromExistingConfig(
+    gc: GlobalTestState,
+    exchangeName: string,
+    opts: { overridePath?: string },
+  ) {
+    const testDir = opts.overridePath ?? gc.testDir;
+    const cfgFilename = testDir + `/exchange-${exchangeName}.conf`;
     const config = Configuration.load(cfgFilename);
     const ec: ExchangeConfig = {
       currency: config.getString("taler", "currency").required(),
@@ -978,7 +1037,9 @@ export class ExchangeService implements ExchangeServiceInterface {
       name: exchangeName,
       roundUnit: config.getString("taler", "currency_round_unit").required(),
     };
-    const privFile = config.getPath("exchange", "master_priv_file").required();
+    const privFile = config
+      .getPath("exchange-offline", "master_priv_file")
+      .required();
     const eddsaPriv = fs.readFileSync(privFile);
     const keyPair: EddsaKeyPair = {
       eddsaPriv,
@@ -1076,11 +1137,13 @@ export class ExchangeService implements ExchangeServiceInterface {
   changeConfig(f: (config: Configuration) => void) {
     const config = Configuration.load(this.configFilename);
     f(config);
-    config.write(this.configFilename);
+    config.write(this.configFilename, { excludeDefaults: true });
   }
 
   static create(gc: GlobalTestState, e: ExchangeConfig) {
+    const testDir = e.overrideTestDir ?? gc.testDir;
     const config = new Configuration();
+    setTalerPaths(config, testDir + "/talerhome");
     config.setString("taler", "currency", e.currency);
     // Required by the exchange but not really used yet.
     config.setString("exchange", "aml_threshold", `${e.currency}:1000000`);
@@ -1089,7 +1152,6 @@ export class ExchangeService implements ExchangeServiceInterface {
       "currency_round_unit",
       e.roundUnit ?? `${e.currency}:0.01`,
     );
-    setTalerPaths(config, gc.testDir + "/talerhome");
     config.setString(
       "exchange",
       "revocation_dir",
@@ -1124,10 +1186,16 @@ export class ExchangeService implements ExchangeServiceInterface {
 
     fs.mkdirSync(path.dirname(masterPrivFile), { recursive: true });
 
+    if (fs.existsSync(masterPrivFile)) {
+      throw new Error(
+        "master priv file already exists, can't create new exchange config",
+      );
+    }
+
     fs.writeFileSync(masterPrivFile, Buffer.from(exchangeMasterKey.eddsaPriv));
 
-    const cfgFilename = gc.testDir + `/exchange-${e.name}.conf`;
-    config.write(cfgFilename);
+    const cfgFilename = testDir + `/exchange-${e.name}.conf`;
+    config.write(cfgFilename, { excludeDefaults: true });
     return new ExchangeService(gc, e, cfgFilename, exchangeMasterKey);
   }
 
@@ -1136,13 +1204,13 @@ export class ExchangeService implements ExchangeServiceInterface {
     offeredCoins.forEach((cc) =>
       setCoin(config, cc(this.exchangeConfig.currency)),
     );
-    config.write(this.configFilename);
+    config.write(this.configFilename, { excludeDefaults: true });
   }
 
   addCoinConfigList(ccs: CoinConfig[]) {
     const config = Configuration.load(this.configFilename);
     ccs.forEach((cc) => setCoin(config, cc));
-    config.write(this.configFilename);
+    config.write(this.configFilename, { excludeDefaults: true });
   }
 
   enableAgeRestrictions(maskStr: string) {
@@ -1153,7 +1221,7 @@ export class ExchangeService implements ExchangeServiceInterface {
       "age_groups",
       maskStr,
     );
-    config.write(this.configFilename);
+    config.write(this.configFilename, { excludeDefaults: true });
   }
 
   get masterPub() {
@@ -1174,7 +1242,7 @@ export class ExchangeService implements ExchangeServiceInterface {
   ): Promise<void> {
     const config = Configuration.load(this.configFilename);
     await f(config);
-    config.write(this.configFilename);
+    config.write(this.configFilename, { excludeDefaults: true });
   }
 
   async addBankAccount(
@@ -1214,7 +1282,7 @@ export class ExchangeService implements ExchangeServiceInterface {
       "password",
       exchangeBankAccount.accountPassword,
     );
-    config.write(this.configFilename);
+    config.write(this.configFilename, { excludeDefaults: true });
   }
 
   exchangeHttpProc: ProcessWrapper | undefined;
@@ -1475,15 +1543,26 @@ export class ExchangeService implements ExchangeServiceInterface {
     );
   }
 
-  async start(): Promise<void> {
-    if (this.isRunning()) {
-      throw Error("exchange is already running");
-    }
+  async dbinit() {
     await sh(
       this.globalState,
       "exchange-dbinit",
       `taler-exchange-dbinit -c "${this.configFilename}"`,
     );
+  }
+
+  async start(
+    opts: { skipDbinit?: boolean; skipKeyup?: boolean } = {},
+  ): Promise<void> {
+    if (this.isRunning()) {
+      throw Error("exchange is already running");
+    }
+
+    const skipDbinit = opts.skipDbinit ?? false;
+
+    if (!skipDbinit) {
+      await this.dbinit();
+    }
 
     this.helperCryptoEddsaProc = this.globalState.spawnService(
       "taler-exchange-secmod-eddsa",
@@ -1514,7 +1593,14 @@ export class ExchangeService implements ExchangeServiceInterface {
     );
 
     await this.pingUntilAvailable();
-    await this.keyup();
+
+    const skipKeyup = opts.skipKeyup ?? false;
+
+    if (!skipKeyup) {
+      await this.keyup();
+    } else {
+      logger.info("skipping keyup");
+    }
   }
 
   async pingUntilAvailable(): Promise<void> {
@@ -1530,6 +1616,7 @@ export interface MerchantConfig {
   currency: string;
   httpPort: number;
   database: string;
+  overrideTestDir?: string;
 }
 
 export interface PrivateOrderStatusQuery {
@@ -1775,8 +1862,13 @@ export interface CreateMerchantTippingReserveRequest {
 }
 
 export class MerchantService implements MerchantServiceInterface {
-  static fromExistingConfig(gc: GlobalTestState, name: string) {
-    const cfgFilename = gc.testDir + `/merchant-${name}.conf`;
+  static fromExistingConfig(
+    gc: GlobalTestState,
+    name: string,
+    opts: { overridePath?: string },
+  ) {
+    const testDir = opts.overridePath ?? gc.testDir;
+    const cfgFilename = testDir + `/merchant-${name}.conf`;
     const config = Configuration.load(cfgFilename);
     const mc: MerchantConfig = {
       currency: config.getString("taler", "currency").required(),
@@ -1846,13 +1938,24 @@ export class MerchantService implements MerchantServiceInterface {
     }
   }
 
-  async start(): Promise<void> {
+  async dbinit() {
     await runCommand(
       this.globalState,
       "merchant-dbinit",
       "taler-merchant-dbinit",
       ["-c", this.configFilename],
     );
+  }
+
+  /**
+   * Start the merchant,
+   */
+  async start(opts: { skipDbinit?: boolean } = {}): Promise<void> {
+    const skipSetup = opts.skipDbinit ?? false;
+
+    if (!skipSetup) {
+      await this.dbinit();
+    }
 
     this.proc = this.globalState.spawnService(
       "taler-merchant-httpd",
@@ -1871,11 +1974,12 @@ export class MerchantService implements MerchantServiceInterface {
     gc: GlobalTestState,
     mc: MerchantConfig,
   ): Promise<MerchantService> {
+    const testDir = mc.overrideTestDir ?? gc.testDir;
     const config = new Configuration();
     config.setString("taler", "currency", mc.currency);
 
-    const cfgFilename = gc.testDir + `/merchant-${mc.name}.conf`;
-    setTalerPaths(config, gc.testDir + "/talerhome");
+    const cfgFilename = testDir + `/merchant-${mc.name}.conf`;
+    setTalerPaths(config, testDir + "/talerhome");
     config.setString("merchant", "serve", "tcp");
     config.setString("merchant", "port", `${mc.httpPort}`);
     config.setString(
@@ -1884,7 +1988,7 @@ export class MerchantService implements MerchantServiceInterface {
       "${TALER_DATA_HOME}/merchant/merchant.priv",
     );
     config.setString("merchantdb-postgres", "config", mc.database);
-    config.write(cfgFilename);
+    config.write(cfgFilename, { excludeDefaults: true });
 
     return new MerchantService(gc, mc, cfgFilename);
   }
@@ -1902,7 +2006,7 @@ export class MerchantService implements MerchantServiceInterface {
       this.merchantConfig.currency,
     );
     config.setString(`merchant-exchange-${e.name}`, "master_key", e.masterPub);
-    config.write(this.configFilename);
+    config.write(this.configFilename, { excludeDefaults: true });
   }
 
   async addDefaultInstance(): Promise<void> {
@@ -1935,14 +2039,8 @@ export class MerchantService implements MerchantServiceInterface {
       name: instanceConfig.name,
       address: instanceConfig.address ?? {},
       jurisdiction: instanceConfig.jurisdiction ?? {},
-      default_max_wire_fee:
-        instanceConfig.defaultMaxWireFee ??
-        `${this.merchantConfig.currency}:1.0`,
-      default_wire_fee_amortization:
-        instanceConfig.defaultWireFeeAmortization ?? 3,
-      default_max_deposit_fee:
-        instanceConfig.defaultMaxDepositFee ??
-        `${this.merchantConfig.currency}:1.0`,
+      // FIXME: In some tests, we might want to make this configurable
+      use_stefan: true,
       default_wire_transfer_delay:
         instanceConfig.defaultWireTransferDelay ??
         Duration.toTalerProtocolDuration(
@@ -1984,9 +2082,6 @@ export interface PartialMerchantInstanceConfig {
   paytoUris: string[];
   address?: unknown;
   jurisdiction?: unknown;
-  defaultMaxWireFee?: string;
-  defaultMaxDepositFee?: string;
-  defaultWireFeeAmortization?: number;
   defaultWireTransferDelay?: TalerProtocolDuration;
   defaultPayDelay?: TalerProtocolDuration;
 }
@@ -2029,9 +2124,7 @@ export interface MerchantInstanceConfig {
   name: string;
   address: unknown;
   jurisdiction: unknown;
-  default_max_wire_fee: string;
-  default_max_deposit_fee: string;
-  default_wire_fee_amortization: number;
+  use_stefan: boolean;
   default_wire_transfer_delay: TalerProtocolDuration;
   default_pay_delay: TalerProtocolDuration;
 }
@@ -2229,12 +2322,15 @@ export class WalletService {
   }
 
   async pingUntilAvailable(): Promise<void> {
+    let nextDelay = backoffStart();
     while (1) {
       try {
         await tryUnixConnect(this.socketPath);
       } catch (e) {
-        logger.info(`connection attempt failed: ${e}`);
-        await delayMs(200);
+        logger.info(`wallet connection attempt failed: ${e}`);
+        logger.info(`waiting on wallet for ${nextDelay}ms`);
+        await delayMs(nextDelay);
+        nextDelay = backoffIncrement(nextDelay);
         continue;
       }
       logger.info("connection to wallet-core succeeded");