From c0dfcf247c0580b43a35824e5deb8bdf0a6b0c67 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Sun, 10 Jan 2021 23:57:06 +0100 Subject: [PATCH] backup schema --- .../src/operations/backup.ts | 29 +-- .../src/operations/exchanges.ts | 1 + .../src/types/backupTypes.ts | 180 ++++++++++++------ .../taler-wallet-core/src/types/dbTypes.ts | 2 + .../taler-wallet-core/src/types/talerTypes.ts | 3 + 5 files changed, 140 insertions(+), 75 deletions(-) diff --git a/packages/taler-wallet-core/src/operations/backup.ts b/packages/taler-wallet-core/src/operations/backup.ts index 72fdf7aa1..0c856b7bc 100644 --- a/packages/taler-wallet-core/src/operations/backup.ts +++ b/packages/taler-wallet-core/src/operations/backup.ts @@ -246,7 +246,7 @@ export async function exportBackup( count: x.count, denom_pub_hash: x.denomPubHash, })), - timestamp_start: wg.timestampStart, + timestamp_created: wg.timestampStart, timestamp_finish: wg.timestampFinish, withdrawal_group_id: wg.withdrawalGroupId, secret_seed: wg.secretSeed, @@ -267,6 +267,8 @@ export async function exportBackup( timestamp_created: reserve.timestampCreated, withdrawal_groups: withdrawalGroupsByReserve[reserve.reservePub] ?? [], + // FIXME! + timestamp_last_activity: reserve.timestampCreated, }; const backupReserves = (backupReservesByExchange[ reserve.exchangeBaseUrl @@ -285,7 +287,7 @@ export async function exportBackup( count: x.count, denom_pub_hash: x.denomPubHash, })), - timestam_picked_up: tip.pickedUpTimestamp, + timestamp_finished: tip.pickedUpTimestamp, timestamp_accepted: tip.acceptedTimestamp, timestamp_created: tip.createdTimestamp, timestamp_expiration: tip.tipExpiration, @@ -296,8 +298,8 @@ export async function exportBackup( await tx.iter(Stores.recoupGroups).forEach((recoupGroup) => { backupRecoupGroups.push({ recoup_group_id: recoupGroup.recoupGroupId, - timestamp_started: recoupGroup.timestampStarted, - timestamp_finished: recoupGroup.timestampFinished, + timestamp_created: recoupGroup.timestampStarted, + timestamp_finish: recoupGroup.timestampFinished, coins: recoupGroup.coinPubs.map((x, i) => ({ coin_pub: x, recoup_finished: recoupGroup.recoupFinishedPerCoin[i], @@ -414,6 +416,7 @@ export async function exportBackup( backupExchanges.push({ base_url: ex.baseUrl, + reserve_closing_delay: ex.details.reserveClosingDelay, accounts: ex.wireInfo.accounts.map((x) => ({ payto_uri: x.payto_uri, master_sig: x.master_sig, @@ -472,7 +475,6 @@ export async function exportBackup( } backupPurchases.push({ - clock_created: 1, contract_terms_raw: purch.download.contractTermsRaw, auto_refund_deadline: purch.autoRefundDeadline, merchant_pay_sig: purch.merchantPaySig, @@ -486,7 +488,6 @@ export async function exportBackup( refunds, timestamp_accept: purch.timestampAccept, timestamp_first_successful_pay: purch.timestampFirstSuccessfulPay, - timestamp_last_refund_status: purch.timestampLastRefundStatus, abort_status: purch.abortStatus === AbortStatus.None ? undefined @@ -564,8 +565,8 @@ export async function exportBackup( backupRefreshGroups.push({ reason: rg.reason as any, refresh_group_id: rg.refreshGroupId, - timestamp_started: rg.timestampCreated, - timestamp_finished: rg.timestampFinished, + timestamp_created: rg.timestampCreated, + timestamp_finish: rg.timestampFinished, old_coins: oldCoins, }); }); @@ -592,6 +593,7 @@ export async function exportBackup( trusted_auditors: {}, trusted_exchanges: {}, intern_table: {}, + error_reports: [], }; // If the backup changed, we increment our clock. @@ -934,6 +936,7 @@ export async function importBackup( wireInfo, details: { currency: backupExchange.currency, + reserveClosingDelay: backupExchange.reserve_closing_delay, auditors: backupExchange.auditors.map((x) => ({ auditor_pub: x.auditor_pub, auditor_url: x.auditor_url, @@ -1102,7 +1105,7 @@ export async function importBackup( reservePub, retryInfo: initRetryInfo(false), secretSeed: backupWg.secret_seed, - timestampStart: backupWg.timestamp_start, + timestampStart: backupWg.timestamp_created, timestampFinish: backupWg.timestamp_finish, withdrawalGroupId: backupWg.withdrawal_group_id, }); @@ -1336,7 +1339,7 @@ export async function importBackup( timestampFirstSuccessfulPay: backupPurchase.timestamp_first_successful_pay, timestampLastRefundStatus: - backupPurchase.timestamp_last_refund_status, + undefined, merchantPaySig: backupPurchase.merchant_pay_sig, lastSessionId: undefined, abortStatus, @@ -1414,8 +1417,8 @@ export async function importBackup( } } await tx.put(Stores.refreshGroups, { - timestampFinished: backupRefreshGroup.timestamp_finished, - timestampCreated: backupRefreshGroup.timestamp_started, + timestampFinished: backupRefreshGroup.timestamp_finish, + timestampCreated: backupRefreshGroup.timestamp_created, refreshGroupId: backupRefreshGroup.refresh_group_id, reason, lastError: undefined, @@ -1452,7 +1455,7 @@ export async function importBackup( lastError: undefined, merchantBaseUrl: backupTip.exchange_base_url, merchantTipId: backupTip.merchant_tip_id, - pickedUpTimestamp: backupTip.timestam_picked_up, + pickedUpTimestamp: backupTip.timestamp_finished, retryInfo: initRetryInfo(false), secretSeed: backupTip.secret_seed, tipAmountEffective: denomsSel.totalCoinValue, diff --git a/packages/taler-wallet-core/src/operations/exchanges.ts b/packages/taler-wallet-core/src/operations/exchanges.ts index 3e71634cd..52da6be62 100644 --- a/packages/taler-wallet-core/src/operations/exchanges.ts +++ b/packages/taler-wallet-core/src/operations/exchanges.ts @@ -212,6 +212,7 @@ async function updateExchangeWithKeys( nextUpdateTime: getExpiryTimestamp(resp, { minDuration: durationFromSpec({ hours: 1 }), }), + reserveClosingDelay: exchangeKeysJson.reserve_closing_delay, }; r.updateStatus = ExchangeUpdateStatus.FetchWire; r.lastError = undefined; diff --git a/packages/taler-wallet-core/src/types/backupTypes.ts b/packages/taler-wallet-core/src/types/backupTypes.ts index 56b50d71c..5570902e6 100644 --- a/packages/taler-wallet-core/src/types/backupTypes.ts +++ b/packages/taler-wallet-core/src/types/backupTypes.ts @@ -20,24 +20,13 @@ * Contains some redundancy with the other type declarations, * as the backup schema must remain very stable and should be self-contained. * - * Current limitations: - * 1. "Ghost spends", where a coin is spent unexpectedly by another wallet - * and a corresponding transaction (that is missing some details!) should - * be added to the transaction history, aren't implemented yet. - * 2. Clocks for denom/coin selections aren't properly modeled yet. - * (Needed for re-denomination of withdrawal / re-selection of coins) - * 3. Preferences about how currencies are to be displayed - * aren't exported yet (and not even implemented in wallet-core). - * 4. Returning money to own bank account isn't supported/exported yet. - * 5. Peer-to-peer payments aren't supported yet. - * 6. Next update time / next auto-refresh time isn't backed up yet. - * 7. Coin/denom selections should be forgettable once that information - * becomes irrelevant. - * 8. Re-denominated payments/refreshes are not shown properly in the total - * payment cost. - * 9. Permanently failed operations aren't properly modeled yet - * 10. Do we somehow need to model the mechanism for first only withdrawing - * the amount to pay the backup provider? + * Future: + * 1. Ghost spends (coin unexpectedly spend by a wallet with shared data) + * 2. Ghost withdrawals (reserve unexpectedly emptied by another wallet with shared data) + * 3. Track losses through re-denomination of payments/refreshes + * 4. (Feature:) Payments to own bank account and P2P-payments need to be backed up + * 5. Track last/next update time, so on restore we need to do less work + * 6. Currency render preferences? * * Questions: * 1. What happens when two backups are merged that have @@ -64,7 +53,7 @@ /** * Imports. */ -import { Timestamp } from "../util/time"; +import { Duration, Timestamp } from "../util/time"; /** * Type alias for strings that are to be treated like amounts. @@ -82,9 +71,12 @@ type BackupAmountString = string; type DeviceIdString = string; /** - * Integer-valued clock. + * Lamport clock timestamp. */ -type ClockValue = number; +export interface ClockStamp { + deviceId: string; + value: number; +} /** * Contract terms JSON. @@ -131,7 +123,7 @@ export interface WalletBackupContentV1 { * tombstones in the hopefully rare case that multiple wallets * are connected to the same sync server. */ - clocks: { [device_id: string]: ClockValue }; + clocks: { [device_id: string]: number }; /** * Timestamp of the backup. @@ -227,6 +219,22 @@ export interface WalletBackupContentV1 { * addresses, etc.) might be shared among many contract terms. */ intern_table: { [hash: string]: any }; + + /** + * Permanent error reports. + */ + error_reports: BackupErrorReport[]; +} + +/** + * Detailed error report. + * + * For auditor-relevant reports with attached cryptographic proof, + * the error report also should contain the submission status to + * the auditor(s). + */ +interface BackupErrorReport { + // FIXME: specify! } /** @@ -253,12 +261,12 @@ export interface BackupTrustAuditor { * Can be undefined if this entry represents a removal delta * from the wallet's defaults. */ - clock_added?: ClockValue; + clock_added?: ClockStamp; /** * Clock for when the auditor trust has been removed. */ - clock_removed?: ClockValue; + clock_removed?: ClockStamp; } /** @@ -286,12 +294,12 @@ export interface BackupTrustExchange { * Can be undefined if this entry represents a removal delta * from the wallet's defaults. */ - clock_added?: ClockValue; + clock_added?: ClockStamp; /** * Clock for when the exchange trust has been removed. */ - clock_removed?: ClockValue; + clock_removed?: ClockStamp; } export class BackupBackupProviderTerms { @@ -347,15 +355,11 @@ export interface BackupRecoupGroup { /** * Timestamp when the recoup was started. */ - timestamp_started: Timestamp; + timestamp_created: Timestamp; - /** - * Timestamp when the recoup finished. - * - * (That means all coins have been recouped and coins to - * be refreshed have been put in a refresh group.) - */ - timestamp_finished: Timestamp | undefined; + timestamp_finish?: Timestamp; + finish_clock?: Timestamp; + finish_is_failure?: boolean; /** * Information about each coin being recouped. @@ -475,10 +479,15 @@ export interface BackupCoin { /** * Does the wallet think that the coin is still fresh? * - * FIXME: If we always refresh when importing a backup, do - * we even need this flag? + * Note that even if a fresh coin is imported, it should still + * be refreshed in most situations. */ fresh: boolean; + + /** + * Clock for the last update to current_amount/fresh. + */ + last_clock?: ClockStamp; } /** @@ -511,11 +520,15 @@ export interface BackupTip { */ timestamp_created: Timestamp; - /** - * Timestamp for when the wallet finished picking up the tip - * from the merchant. - */ - timestam_picked_up: Timestamp | undefined; + timestamp_finished?: Timestamp; + finish_clock?: ClockStamp; + finish_is_failure?: boolean; + + finish_info?: { + timestamp: Timestamp; + clock: ClockStamp; + failure: boolean; + }; /** * The tipped amount. @@ -540,10 +553,9 @@ export interface BackupTip { /** * Selected denominations. Determines the effective tip amount. */ - selected_denoms: { - denom_pub_hash: string; - count: number; - }[]; + selected_denoms: BackupDenomSel; + + selected_denoms_clock?: ClockStamp; } /** @@ -631,12 +643,11 @@ export interface BackupRefreshGroup { */ old_coins: BackupRefreshOldCoin[]; - timestamp_started: Timestamp; + timestamp_created: Timestamp; - /** - * Timestamp when the refresh group finished. - */ - timestamp_finished: Timestamp | undefined; + timestamp_finish?: Timestamp; + finish_clock?: ClockStamp; + finish_is_failure?: boolean; } /** @@ -656,12 +667,11 @@ export interface BackupWithdrawalGroup { * When was the withdrawal operation started started? * Timestamp in milliseconds. */ - timestamp_start: Timestamp; + timestamp_created: Timestamp; - /** - * When was the withdrawal operation completed? - */ timestamp_finish?: Timestamp; + finish_clock?: ClockStamp; + finish_is_failure?: boolean; /** * Amount including fees (i.e. the amount subtracted from the @@ -677,6 +687,8 @@ export interface BackupWithdrawalGroup { * Multiset of denominations selected for withdrawal. */ selected_denoms: BackupDenomSel; + + selected_denoms_clock?: ClockStamp; } export enum BackupRefundState { @@ -725,6 +737,8 @@ export interface BackupRefundItemCommon { * accurately. */ total_refresh_cost_bound: BackupAmountString; + + last_clock?: ClockStamp; } /** @@ -758,11 +772,6 @@ export interface BackupPurchase { */ proposal_id: string; - /** - * Clock when this purchase was created. - */ - clock_created: number; - /** * Contract terms we got from the merchant. */ @@ -791,6 +800,11 @@ export interface BackupPurchase { contribution: BackupAmountString; }[]; + /** + * Clock when the pay coin selection was made/updated. + */ + pay_coins_clock?: ClockStamp; + /** * Total cost initially shown to the user. * @@ -828,10 +842,15 @@ export interface BackupPurchase { refunds: BackupRefundItem[]; /** - * When was the last refund made? - * Set to 0 if no refund was made on the purchase. + * Is the purchase considered defunct (either during payment + * or during abort if abort_status is set). */ - timestamp_last_refund_status: Timestamp | undefined; + defunct?: boolean; + + /** + * Clock for last update to defunct status. + */ + defunct_clock?: ClockStamp; /** * Abort status of the payment. @@ -945,6 +964,21 @@ export interface BackupReserve { */ timestamp_created: Timestamp; + /** + * Timestamp of the last observed activity. + * + * Used to compute when to give up querying the exchange. + */ + timestamp_last_activity: Timestamp; + + /** + * Timestamp of when the reserve closed. + * + * Note that the last activity can be after the closing time + * due to recouping. + */ + timestamp_closed?: Timestamp; + /** * Wire information (as payto URI) for the bank account that * transfered funds for this reserve. @@ -1012,6 +1046,9 @@ export interface BackupReserve { * Groups of withdrawal operations for this reserve. Typically just one. */ withdrawal_groups: BackupWithdrawalGroup[]; + + defective?: boolean; + defective_clock?: ClockStamp; } /** @@ -1134,6 +1171,11 @@ export interface BackupExchange { */ protocol_version: string; + /** + * Closing delay of reserves. + */ + reserve_closing_delay: Duration; + /** * Signing keys we got from the exchange, can also contain * older signing keys that are not returned by /keys anymore. @@ -1159,6 +1201,18 @@ export interface BackupExchange { * ETag for last terms of service download. */ tos_etag_accepted: string | undefined; + + /** + * Clock value of the last update. + */ + last_clock?: ClockStamp; + + /** + * Should this exchange be considered defective? + */ + defective?: boolean; + + defective_clock?: ClockStamp; } export enum BackupProposalStatus { @@ -1236,6 +1290,8 @@ export interface BackupProposal { */ proposal_status: BackupProposalStatus; + proposal_status_clock?: ClockStamp; + /** * Proposal that this one got "redirected" to as part of * the repurchase detection. diff --git a/packages/taler-wallet-core/src/types/dbTypes.ts b/packages/taler-wallet-core/src/types/dbTypes.ts index 551495a68..62ad01d25 100644 --- a/packages/taler-wallet-core/src/types/dbTypes.ts +++ b/packages/taler-wallet-core/src/types/dbTypes.ts @@ -392,6 +392,8 @@ export interface ExchangeDetails { */ protocolVersion: string; + reserveClosingDelay: Duration; + /** * Signing keys we got from the exchange, can also contain * older signing keys that are not returned by /keys anymore. diff --git a/packages/taler-wallet-core/src/types/talerTypes.ts b/packages/taler-wallet-core/src/types/talerTypes.ts index fe30fa8b9..b17c101b0 100644 --- a/packages/taler-wallet-core/src/types/talerTypes.ts +++ b/packages/taler-wallet-core/src/types/talerTypes.ts @@ -672,6 +672,8 @@ export class ExchangeKeysJson { * Protocol version. */ version: string; + + reserve_closing_delay: Duration; } /** @@ -1193,6 +1195,7 @@ export const codecForExchangeKeysJson = (): Codec => .property("recoup", codecOptional(codecForList(codecForRecoup()))) .property("signkeys", codecForList(codecForExchangeSigningKey())) .property("version", codecForString()) + .property("reserve_closing_delay", codecForDuration) .build("KeysJson"); export const codecForWireFeesJson = (): Codec =>