implement new GNUnet config features

packages/taler-util/package.json

@@ -23,6 +23,7 @@
   "private": false,
   "scripts": {
     "prepare": "tsc",
+    "compile": "tsc",
     "test": "tsc && ava",
     "clean": "rimraf dist lib tsconfig.tsbuildinfo",
     "pretty": "prettier --write src"

packages/taler-util/src/talerconfig.ts

@@ -41,6 +41,36 @@ const nodejs_fs = (function () {
   };
 })();
 
+const nodejs_path = (function () {
+  let path: typeof import("path");
+  return function () {
+    if (!path) {
+      /**
+       * need to use an expression when doing a require if we want
+       * webpack not to find out about the requirement
+       */
+      const _r = "require";
+      path = module[_r]("path");
+    }
+    return path;
+  };
+})();
+
+const nodejs_os = (function () {
+  let os: typeof import("os");
+  return function () {
+    if (!os) {
+      /**
+       * need to use an expression when doing a require if we want
+       * webpack not to find out about the requirement
+       */
+      const _r = "require";
+      os = module[_r]("os");
+    }
+    return os;
+  };
+})();
+
 export class ConfigError extends Error {
   constructor(message: string) {
     super();
@@ -50,8 +80,19 @@ export class ConfigError extends Error {
   }
 }
 
-type OptionMap = { [optionName: string]: string };
+interface Entry {
-type SectionMap = { [sectionName: string]: OptionMap };
+  value: string;
+  sourceLine: number;
+  sourceFile: string;
+}
+
+interface Section {
+  secretFilename?: string;
+  inaccessible: boolean;
+  entries: { [optionName: string]: Entry };
+}
+
+type SectionMap = { [sectionName: string]: Section };
 
 export class ConfigValue<T> {
   constructor(
@@ -91,6 +132,20 @@ export class ConfigValue<T> {
   }
 }
 
+/**
+ * Expand a path by resolving the tilde syntax for home directories
+ * and by making relative paths absolute based on the current working directory.
+ */
+export function expandPath(path: string): string {
+  if (path[0] === "~") {
+    path = nodejs_path().join(nodejs_os().homedir(), path.slice(1));
+  }
+  if (path[0] !== "/") {
+    path = nodejs_path().join(process.cwd(), path);
+  }
+  return path;
+}
+
 /**
  * Shell-style path substitution.
  *
@@ -174,32 +229,246 @@ export function pathsub(
   return s;
 }
 
+export interface LoadOptions {
+  filename?: string;
+  banDirectives?: boolean;
+}
+
+export interface StringifyOptions {
+  diagnostics?: boolean;
+}
+
+export interface LoadedFile {
+  filename: string;
+  level: number;
+}
+
+/**
+ * Check for a simple wildcard match.
+ * Only asterisks are allowed.
+ * Asterisks match everything, including slashes.
+ *
+ * @param pattern pattern with wildcards
+ * @param str string to match against
+ * @returns true on match, false otherwise
+ */
+function globMatch(pattern: string, str: string): boolean {
+  /* Position in the input string */
+  let strPos = 0;
+  /* Position in the pattern */
+  let patPos = 0;
+  /* Backtrack position in string */
+  let strBt = -1;
+  /* Backtrack position in pattern */
+  let patBt = -1;
+
+  for (;;) {
+    if (pattern[patPos] === "*") {
+      strBt = strPos;
+      patBt = patPos++;
+    } else if (patPos === pattern.length && strPos === str.length) {
+      return true;
+    } else if (pattern[patPos] === str[strPos]) {
+      strPos++;
+      patPos++;
+    } else {
+      if (patBt < 0) {
+        return false;
+      }
+      strPos = strBt + 1;
+      if (strPos >= str.length) {
+        return false;
+      }
+      patPos = patBt;
+    }
+  }
+}
+
+function normalizeInlineFilename(parentFile: string, f: string): string {
+  if (f[0] === "/") {
+    return f;
+  }
+  const resolvedParentDir = nodejs_path().dirname(
+    nodejs_fs().realpathSync(parentFile),
+  );
+  return nodejs_path().join(resolvedParentDir, f);
+}
+
 export class Configuration {
   private sectionMap: SectionMap = {};
 
-  loadFromString(s: string): void {
+  private hintEntrypoint: string | undefined;
+
+  private loadedFiles: LoadedFile[] = [];
+
+  private nestLevel = 0;
+
+  loadFromFilename(filename: string, opts: LoadOptions = {}): void {
+    filename = expandPath(filename);
+
+    const checkCycle = () => {
+      let level = this.nestLevel;
+      const fns = [...this.loadedFiles].reverse();
+      for (const lf of fns) {
+        if (lf.level >= level) {
+          continue;
+        }
+        level = lf.level;
+        if (lf.filename === filename) {
+          throw Error(`cyclic inline ${lf.filename} -> ${filename}`);
+        }
+      }
+    };
+
+    checkCycle();
+
+    const s = nodejs_fs().readFileSync(filename, "utf-8");
+    this.loadedFiles.push({
+      filename: filename,
+      level: this.nestLevel,
+    });
+    const oldNestLevel = this.nestLevel;
+    this.nestLevel += 1;
+    try {
+      this.loadFromString(s, {
+        ...opts,
+        filename: filename,
+      });
+    } finally {
+      this.nestLevel = oldNestLevel;
+    }
+  }
+
+  loadGlob(parentFilename: string, fileglob: string): void {
+    const resolvedParent = nodejs_fs().realpathSync(parentFilename);
+    const parentDir = nodejs_path().dirname(resolvedParent);
+
+    let fullFileglob: string;
+
+    if (fileglob.startsWith("/")) {
+      fullFileglob = fileglob;
+    } else {
+      fullFileglob = nodejs_path().join(parentDir, fileglob);
+    }
+
+    fullFileglob = expandPath(fullFileglob);
+
+    const head = nodejs_path().dirname(fullFileglob);
+    const tail = nodejs_path().basename(fullFileglob);
+
+    const files = nodejs_fs().readdirSync(head);
+    for (const f of files) {
+      if (globMatch(tail, f)) {
+        const fullPath = nodejs_path().join(head, f);
+        this.loadFromFilename(fullPath);
+      }
+    }
+  }
+
+  private loadSecret(sectionName: string, filename: string): void {
+    const sec = this.provideSection(sectionName);
+    sec.secretFilename = filename;
+    const otherCfg = new Configuration();
+    try {
+      nodejs_fs().accessSync(filename, nodejs_fs().constants.R_OK);
+    } catch (err) {
+      sec.inaccessible = true;
+      return;
+    }
+    otherCfg.loadFromFilename(filename, {
+      banDirectives: true,
+    });
+    const otherSec = otherCfg.provideSection(sectionName);
+    for (const opt of Object.keys(otherSec.entries)) {
+      this.setString(sectionName, opt, otherSec.entries[opt].value);
+    }
+  }
+
+  loadFromString(s: string, opts: LoadOptions = {}): void {
+    let lineNo = 0;
+    const fn = opts.filename ?? "<input>";
     const reComment = /^\s*#.*$/;
     const reSection = /^\s*\[\s*([^\]]*)\s*\]\s*$/;
     const reParam = /^\s*([^=]+?)\s*=\s*(.*?)\s*$/;
+    const reDirective = /^\s*@([a-zA-Z-_]+)@\s*(.*?)\s*$/;
     const reEmptyLine = /^\s*$/;
 
     let currentSection: string | undefined = undefined;
 
     const lines = s.split("\n");
     for (const line of lines) {
+      lineNo++;
       if (reEmptyLine.test(line)) {
         continue;
       }
       if (reComment.test(line)) {
         continue;
       }
+      const directiveMatch = line.match(reDirective);
+      if (directiveMatch) {
+        if (opts.banDirectives) {
+          throw Error(
+            `invalid configuration, directive in ${fn}:${lineNo} forbidden`,
+          );
+        }
+        const directive = directiveMatch[1].toLowerCase();
+        switch (directive) {
+          case "inline": {
+            if (!opts.filename) {
+              throw Error(
+                `invalid configuration, @inline-matching@ directive in ${fn}:${lineNo} can only be used from a file`,
+              );
+            }
+            const arg = directiveMatch[2].trim();
+            this.loadFromFilename(normalizeInlineFilename(opts.filename, arg));
+            break;
+          }
+          case "inline-secret": {
+            if (!opts.filename) {
+              throw Error(
+                `invalid configuration, @inline-matching@ directive in ${fn}:${lineNo} can only be used from a file`,
+              );
+            }
+            const arg = directiveMatch[2].trim();
+            const sp = arg.split(" ").map((x) => x.trim());
+            if (sp.length != 2) {
+              throw Error(
+                `invalid configuration, @inline-secret@ directive in ${fn}:${lineNo} requires two arguments`,
+              );
+            }
+            const secretFilename = normalizeInlineFilename(
+              opts.filename,
+              sp[1],
+            );
+            this.loadSecret(sp[0], secretFilename);
+            break;
+          }
+          case "inline-matching": {
+            const arg = directiveMatch[2].trim();
+            if (!opts.filename) {
+              throw Error(
+                `invalid configuration, @inline-matching@ directive in ${fn}:${lineNo} can only be used from a file`,
+              );
+            }
+            this.loadGlob(opts.filename, arg);
+            break;
+          }
+          default:
+            throw Error(
+              `invalid configuration, unsupported directive in ${fn}:${lineNo}`,
+            );
+        }
+        continue;
+      }
       const secMatch = line.match(reSection);
       if (secMatch) {
         currentSection = secMatch[1];
         continue;
       }
       if (currentSection === undefined) {
-        throw Error("invalid configuration, expected section header");
+        throw Error(
+          `invalid configuration, expected section header in ${fn}:${lineNo}`,
+        );
       }
       currentSection = currentSection.toUpperCase();
       const paramMatch = line.match(reParam);
@@ -209,22 +478,46 @@ export class Configuration {
         if (val.startsWith('"') && val.endsWith('"')) {
           val = val.slice(1, val.length - 1);
         }
-        const sec = this.sectionMap[currentSection] ?? {};
+        const sec = this.provideSection(currentSection);
-        this.sectionMap[currentSection] = Object.assign(sec, {
+        sec.entries[optName] = {
-          [optName]: val,
+          value: val,
-        });
+          sourceFile: opts.filename ?? "<unknown>",
+          sourceLine: lineNo,
+        };
         continue;
       }
       throw Error(
-        "invalid configuration, expected section header or option assignment",
+        `invalid configuration, expected section header, option assignment or directive in ${fn}:${lineNo}`,
       );
     }
   }
 
-  setString(section: string, option: string, value: string): void {
+  private provideSection(section: string): Section {
     const secNorm = section.toUpperCase();
-    const sec = this.sectionMap[secNorm] ?? (this.sectionMap[secNorm] = {});
+    if (this.sectionMap[secNorm]) {
-    sec[option.toUpperCase()] = value;
+      return this.sectionMap[secNorm];
+    }
+    const newSec: Section = {
+      entries: {},
+      inaccessible: false,
+    };
+    this.sectionMap[secNorm] = newSec;
+    return newSec;
+  }
+
+  private findEntry(section: string, option: string): Entry | undefined {
+    const secNorm = section.toUpperCase();
+    const optNorm = option.toUpperCase();
+    return this.sectionMap[secNorm]?.entries[optNorm];
+  }
+
+  setString(section: string, option: string, value: string): void {
+    const sec = this.provideSection(section);
+    sec.entries[option.toUpperCase()] = {
+      value,
+      sourceLine: 0,
+      sourceFile: "<unknown>",
+    };
   }
 
   /**
@@ -237,14 +530,14 @@ export class Configuration {
   getString(section: string, option: string): ConfigValue<string> {
     const secNorm = section.toUpperCase();
     const optNorm = option.toUpperCase();
-    const val = (this.sectionMap[secNorm] ?? {})[optNorm];
+    const val = this.findEntry(secNorm, optNorm)?.value;
     return new ConfigValue(secNorm, optNorm, val, (x) => x);
   }
 
   getPath(section: string, option: string): ConfigValue<string> {
     const secNorm = section.toUpperCase();
     const optNorm = option.toUpperCase();
-    const val = (this.sectionMap[secNorm] ?? {})[optNorm];
+    const val = this.findEntry(secNorm, optNorm)?.value;
     return new ConfigValue(secNorm, optNorm, val, (x) =>
       pathsub(x, (v, d) => this.lookupVariable(v, d + 1)),
     );
@@ -253,7 +546,7 @@ export class Configuration {
   getYesNo(section: string, option: string): ConfigValue<boolean> {
     const secNorm = section.toUpperCase();
     const optNorm = option.toUpperCase();
-    const val = (this.sectionMap[secNorm] ?? {})[optNorm];
+    const val = this.findEntry(secNorm, optNorm)?.value;
     const convert = (x: string): boolean => {
       x = x.toLowerCase();
       if (x === "yes") {
@@ -271,7 +564,7 @@ export class Configuration {
   getNumber(section: string, option: string): ConfigValue<number> {
     const secNorm = section.toUpperCase();
     const optNorm = option.toUpperCase();
-    const val = (this.sectionMap[secNorm] ?? {})[optNorm];
+    const val = this.findEntry(secNorm, optNorm)?.value;
     const convert = (x: string): number => {
       try {
         return Number.parseInt(x, 10);
@@ -287,7 +580,7 @@ export class Configuration {
   lookupVariable(x: string, depth: number = 0): string | undefined {
     // We loop up options in PATHS in upper case, as option names
     // are case insensitive
-    const val = (this.sectionMap["PATHS"] ?? {})[x.toUpperCase()];
+    const val = this.findEntry("PATHS", x)?.value;
     if (val !== undefined) {
       return pathsub(val, (v, d) => this.lookupVariable(v, d), depth);
     }
@@ -300,33 +593,105 @@ export class Configuration {
   }
 
   getAmount(section: string, option: string): ConfigValue<AmountJson> {
-    const val = (this.sectionMap[section] ?? {})[option];
+    const val = (
-    return new ConfigValue(section, option, val, (x) =>
+      this.sectionMap[section] ?? {
+        entries: {},
+      }
+    ).entries[option];
+    return new ConfigValue(section, option, val.value, (x) =>
       Amounts.parseOrThrow(x),
     );
   }
 
-  static load(filename: string): Configuration {
+  loadFrom(dirname: string): void {
-    const s = nodejs_fs().readFileSync(filename, "utf-8");
+    const files = nodejs_fs().readdirSync(dirname);
+    for (const f of files) {
+      const fn = nodejs_path().join(dirname, f);
+      this.loadFromFilename(fn);
+    }
+  }
+
+  private loadDefaults(): void {
+    let bc = process.env["TALER_BASE_CONFIG"];
+    if (!bc) {
+      bc = "/usr/share/taler/config.d";
+    }
+    this.loadFrom(bc);
+  }
+
+  getDefaultConfigFilename(): string | undefined {
+    const xdg = process.env["XDG_CONFIG_HOME"];
+    const home = process.env["HOME"];
+    let fn: string | undefined;
+    if (xdg) {
+      fn = nodejs_path().join(xdg, "taler.conf");
+    } else if (home) {
+      fn = nodejs_path().join(home, ".config/taler.conf");
+    }
+    if (fn && nodejs_fs().existsSync(fn)) {
+      return fn;
+    }
+    const etc1 = "/etc/taler.conf";
+    if (nodejs_fs().existsSync(etc1)) {
+      return etc1;
+    }
+    const etc2 = "/etc/taler/taler.conf";
+    if (nodejs_fs().existsSync(etc2)) {
+      return etc2;
+    }
+    return undefined;
+  }
+
+  static load(filename?: string): Configuration {
     const cfg = new Configuration();
-    cfg.loadFromString(s);
+    cfg.loadDefaults();
+    if (filename) {
+      cfg.loadFromFilename(filename);
+    } else {
+      const fn = cfg.getDefaultConfigFilename();
+      if (fn) {
+        cfg.loadFromFilename(fn);
+      }
+    }
+    cfg.hintEntrypoint = filename;
     return cfg;
   }
 
-  write(filename: string): void {
+  stringify(opts: StringifyOptions = {}): string {
     let s = "";
+    if (opts.diagnostics) {
+      s += "# Configuration file diagnostics\n";
+      s += "#\n";
+      s += `# Entry point: ${this.hintEntrypoint ?? "<none>"}\n`;
+      s += "#\n";
+      s += "# Loaded files:\n";
+      for (const f of this.loadedFiles) {
+        s += `# ${f.filename}\n`;
+      }
+      s += "#\n\n";
+    }
     for (const sectionName of Object.keys(this.sectionMap)) {
+      const sec = this.sectionMap[sectionName];
+      if (opts.diagnostics && sec.secretFilename) {
+        s += `# Secret section from ${sec.secretFilename}\n`;
+        s += `# Secret accessible: ${!sec.inaccessible}\n`;
+      }
       s += `[${sectionName}]\n`;
-      for (const optionName of Object.keys(
+      for (const optionName of Object.keys(sec.entries)) {
-        this.sectionMap[sectionName] ?? {},
+        const entry = this.sectionMap[sectionName].entries[optionName];
-      )) {
+        if (entry !== undefined) {
-        const val = this.sectionMap[sectionName][optionName];
+          if (opts.diagnostics) {
-        if (val !== undefined) {
+            s += `# ${entry.sourceFile}:${entry.sourceLine}\n`;
-          s += `${optionName} = ${val}\n`;
+          }
+          s += `${optionName} = ${entry.value}\n`;
         }
       }
       s += "\n";
     }
-    nodejs_fs().writeFileSync(filename, s);
+    return s;
+  }
+
+  write(filename: string): void {
+    nodejs_fs().writeFileSync(filename, this.stringify());
   }
 }

packages/taler-wallet-cli/src/index.ts

@@ -873,9 +873,18 @@ const deploymentConfigCli = deploymentCli.subcommand("configArgs", "config", {
   help: "Subcommands the Taler configuration.",
 });
 
-deploymentConfigCli.subcommand("show", "show").action(async (args) => {
+deploymentConfigCli
-  const cfg = new Configuration();
+  .subcommand("show", "show")
-});
+  .flag("diagnostics", ["-d", "--diagnostics"])
+  .maybeArgument("cfgfile", clk.STRING, {})
+  .action(async (args) => {
+    const cfg = Configuration.load(args.show.cfgfile);
+    console.log(
+      cfg.stringify({
+        diagnostics: args.show.diagnostics,
+      }),
+    );
+  });
 
 const testCli = walletCli.subcommand("testingArgs", "testing", {
   help: "Subcommands for testing.",