Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: integrate persistent cache with blockchain providers #1697

Merged
merged 1 commit into from
Feb 13, 2025
Merged

feat: integrate persistent cache with blockchain providers #1697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions apps/browser-extension-wallet/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@
},
"dependencies": {
"@ant-design/icons": "^4.7.0",
"@cardano-sdk/cardano-services-client": "0.26.2",
"@cardano-sdk/core": "0.45.1",
"@cardano-sdk/dapp-connector": "0.13.4",
"@cardano-sdk/input-selection": "0.14.2",
"@cardano-sdk/tx-construction": "0.26.1",
"@cardano-sdk/util": "0.15.6",
"@cardano-sdk/util-rxjs": "0.9.5",
"@cardano-sdk/wallet": "0.51.9",
"@cardano-sdk/web-extension": "0.38.10",
"@cardano-sdk/cardano-services-client": "0.26.3",
"@cardano-sdk/core": "0.45.2",
"@cardano-sdk/dapp-connector": "0.13.5",
"@cardano-sdk/input-selection": "0.14.3",
"@cardano-sdk/tx-construction": "0.26.2",
"@cardano-sdk/util": "0.15.7",
"@cardano-sdk/util-rxjs": "0.9.6",
"@cardano-sdk/wallet": "0.51.10",
"@cardano-sdk/web-extension": "0.38.14",
"@emurgo/cip14-js": "~3.0.1",
"@input-output-hk/lace-ui-toolkit": "3.2.1",
"@lace/cardano": "0.1.0",
Expand Down Expand Up @@ -104,7 +104,7 @@
"zustand": "3.5.14"
},
"devDependencies": {
"@cardano-sdk/hardware-ledger": "0.15.2",
"@cardano-sdk/hardware-ledger": "0.15.3",
"@emurgo/cardano-message-signing-asmjs": "1.0.1",
"@openpgp/web-stream-tools": "0.0.11-patch-0",
"@pdfme/common": "^4.0.2",
Expand Down
4 changes: 3 additions & 1 deletion apps/browser-extension-wallet/src/lib/scripts/background/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { storage } from 'webextension-polyfill';
import axiosFetchAdapter from '@shiroyasha9/axios-fetch-adapter';
import { Wallet } from '@lace/cardano';
import { RemoteApiProperties, RemoteApiPropertyType } from '@cardano-sdk/web-extension';
Expand Down Expand Up @@ -62,7 +63,8 @@ export const getProviders = async (chainName: Wallet.ChainName): Promise<Wallet.
logger,
experiments: {
useWebSocket: isExperimentEnabled(ExperimentName.WEBSOCKET_API)
}
},
extensionLocalStorage: storage.local
});
};

Expand Down
22 changes: 11 additions & 11 deletions packages/cardano/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,16 @@
"watch": "yarn build --watch"
},
"dependencies": {
"@cardano-sdk/cardano-services-client": "0.26.2",
"@cardano-sdk/core": "0.45.1",
"@cardano-sdk/crypto": "0.2.1",
"@cardano-sdk/hardware-ledger": "0.15.2",
"@cardano-sdk/hardware-trezor": "0.7.1",
"@cardano-sdk/key-management": "0.27.1",
"@cardano-sdk/tx-construction": "0.26.1",
"@cardano-sdk/util": "0.15.6",
"@cardano-sdk/wallet": "0.51.9",
"@cardano-sdk/web-extension": "0.38.10",
"@cardano-sdk/cardano-services-client": "0.26.3",
"@cardano-sdk/core": "0.45.2",
"@cardano-sdk/crypto": "0.2.2",
"@cardano-sdk/hardware-ledger": "0.15.3",
"@cardano-sdk/hardware-trezor": "0.7.2",
"@cardano-sdk/key-management": "0.27.2",
"@cardano-sdk/tx-construction": "0.26.2",
"@cardano-sdk/util": "0.15.7",
"@cardano-sdk/wallet": "0.51.10",
"@cardano-sdk/web-extension": "0.38.14",
"@lace/common": "0.1.0",
"@ledgerhq/devices": "^8.4.4",
"@stablelib/chacha20poly1305": "1.0.1",
Expand All @@ -73,7 +73,7 @@
},
"devDependencies": {
"@blockfrost/blockfrost-js": "^5.5.0",
"@cardano-sdk/util-dev": "0.25.4",
"@cardano-sdk/util-dev": "0.25.5",
"@emurgo/cardano-message-signing-browser": "1.0.1",
"@types/webextension-polyfill": "0.10.0",
"axios": "^1.7.4",
Expand Down
20 changes: 19 additions & 1 deletion packages/cardano/src/wallet/lib/__tests__/blockfrost-input-resolver.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,25 @@ describe('BlockfrostInputResolver', () => {
warn: jest.fn()
} as unknown as jest.Mocked<Logger>;

resolver = new BlockfrostInputResolver(clientMock, loggerMock);
// eslint-disable-next-line unicorn/consistent-function-scoping
const createProviderCache = () => {
const cache = new Map();
return {
async get(key: string) {
return cache.get(key);
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
async set(key: string, val: any) {
cache.set(key, val);
}
};
};

resolver = new BlockfrostInputResolver({
cache: createProviderCache(),
client: clientMock,
logger: loggerMock
});
});

afterEach(() => {
Expand Down
24 changes: 17 additions & 7 deletions packages/cardano/src/wallet/lib/blockfrost-input-resolver.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/* eslint-disable unicorn/no-null, @typescript-eslint/no-non-null-assertion */
import { Cardano } from '@cardano-sdk/core';
import { BlockfrostClient, BlockfrostError, BlockfrostToCore } from '@cardano-sdk/cardano-services-client';
import type { Cache } from '@cardano-sdk/util';
import { Logger } from 'ts-log';
import { Responses } from '@blockfrost/blockfrost-js';

Expand All @@ -14,21 +15,29 @@ const NOT_FOUND_STATUS = 404;
*/
const txInToId = (txIn: Cardano.TxIn): string => `${txIn.txId}#${txIn.index}`;

type BlockfrostInputResolverDependencies = {
cache: Cache<Cardano.TxOut>;
client: BlockfrostClient;
logger: Logger;
};

/**
* A resolver class to fetch and resolve transaction inputs using Blockfrost API.
*/
export class BlockfrostInputResolver implements Cardano.InputResolver {
readonly #logger: Logger;
readonly #client: BlockfrostClient;
readonly #txCache = new Map<string, Cardano.TxOut>();
readonly #txCache: Cache<Cardano.TxOut>;

/**
* Constructs a new BlockfrostInputResolver.
*
* @param cache - A caching interface.
* @param client - The Blockfrost client instance to interact with the Blockfrost API.
* @param logger - The logger instance to log messages to.
*/
constructor(client: BlockfrostClient, logger: Logger) {
constructor({ cache, client, logger }: BlockfrostInputResolverDependencies) {
this.#txCache = cache;
this.#client = client;
this.#logger = logger;
}
Expand All @@ -44,9 +53,10 @@ export class BlockfrostInputResolver implements Cardano.InputResolver {
public async resolveInput(input: Cardano.TxIn, options?: Cardano.ResolveOptions): Promise<Cardano.TxOut | null> {
this.#logger.debug(`Resolving input ${input.txId}#${input.index}`);

if (this.#txCache.has(txInToId(input))) {
const cached = await this.#txCache.get(txInToId(input));
if (cached) {
this.#logger.debug(`Resolved input ${input.txId}#${input.index} from cache`);
return this.#txCache.get(txInToId(input))!;
return cached;
}

const resolved = this.resolveFromHints(input, options);
Expand All @@ -69,7 +79,7 @@ export class BlockfrostInputResolver implements Cardano.InputResolver {
for (const hint of options.hints.transactions) {
if (input.txId === hint.id && hint.body.outputs.length > input.index) {
this.#logger.debug(`Resolved input ${input.txId}#${input.index} from hint`);
this.#txCache.set(txInToId(input), hint.body.outputs[input.index]);
void this.#txCache.set(txInToId(input), hint.body.outputs[input.index]);

return hint.body.outputs[input.index];
}
Expand All @@ -80,7 +90,7 @@ export class BlockfrostInputResolver implements Cardano.InputResolver {
for (const utxo of options.hints.utxos) {
if (input.txId === utxo[0].txId && input.index === utxo[0].index) {
this.#logger.debug(`Resolved input ${input.txId}#${input.index} from hint`);
this.#txCache.set(txInToId(input), utxo[1]);
void this.#txCache.set(txInToId(input), utxo[1]);

return utxo[1];
}
Expand Down Expand Up @@ -119,7 +129,7 @@ export class BlockfrostInputResolver implements Cardano.InputResolver {

const coreTxOut = BlockfrostToCore.txOut(blockfrostUTxO);

this.#txCache.set(txInToId(txIn), coreTxOut);
void this.#txCache.set(txInToId(txIn), coreTxOut);

this.#logger.debug(`Resolved input ${txIn.txId}#${txIn.index} from Blockfrost`);
return coreTxOut;
Expand Down
70 changes: 65 additions & 5 deletions packages/cardano/src/wallet/lib/providers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable no-new, complexity, sonarjs/cognitive-complexity */
import { Storage } from 'webextension-polyfill';
import { AxiosAdapter } from 'axios';
import { Logger } from 'ts-log';
import {
Expand Down Expand Up @@ -33,7 +34,7 @@ import {
BlockfrostNetworkInfoProvider,
BlockfrostRewardAccountInfoProvider
} from '@cardano-sdk/cardano-services-client';
import { RemoteApiProperties, RemoteApiPropertyType } from '@cardano-sdk/web-extension';
import { RemoteApiProperties, RemoteApiPropertyType, createPersistentCacheStorage } from '@cardano-sdk/web-extension';
import { BlockfrostAddressDiscovery } from '@wallet/lib/blockfrost-address-discovery';
import { WalletProvidersDependencies } from './cardano-wallet';
import { BlockfrostInputResolver } from './blockfrost-input-resolver';
Expand Down Expand Up @@ -86,6 +87,7 @@ interface ProvidersConfig {
experiments: {
useWebSocket?: boolean;
};
extensionLocalStorage: Storage.LocalStorageArea;
}

/**
Expand All @@ -94,11 +96,41 @@ interface ProvidersConfig {
* If a new one needs to be created (ex. on network change) the previous instance needs to be closed. */
let wsProvider: CardanoWsClient;

enum CacheName {
chainHistoryProvider = 'chain-history-provider-cache',
inputResolver = 'input-resolver-cache',
utxoProvider = 'utxo-provider-cache'
}

// eslint-disable-next-line no-magic-numbers
const sizeOf1mb = 1024 * 1024;

// The count values have been calculated by filling the cache by impersonating a few
// rich wallets and then getting the average size of a single item per each cache collection
const cacheAssignment: Record<CacheName, { count: number; size: number }> = {
[CacheName.chainHistoryProvider]: {
count: 5_180_160_021,
// eslint-disable-next-line no-magic-numbers
size: 30 * sizeOf1mb
},
[CacheName.inputResolver]: {
count: 65_529_512_340,
// eslint-disable-next-line no-magic-numbers
size: 30 * sizeOf1mb
},
[CacheName.utxoProvider]: {
count: 6_530_251_302,
// eslint-disable-next-line no-magic-numbers
size: 30 * sizeOf1mb
}
Copy link
Contributor Author

@szymonmaslowski szymonmaslowski Feb 6, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Values have been calculated with first populating data to cache running lace and then running this script and passing a 7 (mb) as a parameter representing total space expected to occupy by the 3 caches. I think we should run it for a few different wallet to see how their states project to those cache sizes.

script 
const getSizesPerCollection = async (totalMaxSize) => {
  const chainHistoryProviderMetadataKey = 'chain-history-provider-cache-metadata';
  const inputReslverMetadataKey = 'input-resolver-cache-metadata';
  const utxoProviderMetadataKey = 'utxo-provider-cache-metadata';

  const getSize = async (metadataKey) => {
    const { [metadataKey]: metadata } = await chrome.storage.local.get(metadataKey);
    const bytes = await chrome.storage.local.getBytesInUse([...Object.keys(metadata), metadataKey]);
    const itemsCount = Object.keys(metadata).length;
    const sizeInMb = bytes / 1024 / 1024;
    const sizeOfSingleItem = sizeInMb / itemsCount;

    return {
      sizeInMb,
      sizeOfSingleItem
    };
  };

  const chainHistorySize = await getSize(chainHistoryProviderMetadataKey);
  const inputResolverSize = await getSize(inputReslverMetadataKey);
  const utxoSize = await getSize(utxoProviderMetadataKey);

  const totalSize = chainHistorySize.sizeInMb + inputResolverSize.sizeInMb + utxoSize.sizeInMb;

  const targetSizeOfChainHistory = totalMaxSize * (chainHistorySize.sizeInMb / totalSize);
  const targetSizeOfInputResolver = totalMaxSize * (inputResolverSize.sizeInMb / totalSize);
  const targetSizeOfUtxo = totalMaxSize * (utxoSize.sizeInMb / totalSize);

  const targetCountOfChainHistory = targetSizeOfChainHistory / chainHistorySize.sizeOfSingleItem;
  const targetCountOfInputResolver = targetSizeOfInputResolver / inputResolverSize.sizeOfSingleItem;
  const targetCountOfUtxo = targetSizeOfUtxo / utxoSize.sizeOfSingleItem;

  return {
    chainHistory: {
      count: targetCountOfChainHistory,
      size: targetSizeOfChainHistory
    },
    inputResolver: {
      count: targetCountOfInputResolver,
      size: targetSizeOfInputResolver
    },
    utxo: {
      count: targetCountOfUtxo,
      size: targetSizeOfUtxo
    }
  };
};

};

export const createProviders = ({
axiosAdapter,
env: { baseCardanoServicesUrl: baseUrl, customSubmitTxUrl, blockfrostConfig },
logger = console,
experiments: { useWebSocket }
experiments: { useWebSocket },
extensionLocalStorage
}: ProvidersConfig): WalletProvidersDependencies => {
const httpProviderConfig: CreateHttpProviderConfig<Provider> = { baseUrl, logger, adapter: axiosAdapter };

Expand All @@ -107,7 +139,17 @@ export const createProviders = ({
});
const assetProvider = new BlockfrostAssetProvider(blockfrostClient, logger);
const networkInfoProvider = new BlockfrostNetworkInfoProvider(blockfrostClient, logger);
const chainHistoryProvider = new BlockfrostChainHistoryProvider(blockfrostClient, networkInfoProvider, logger);
const chainHistoryProvider = new BlockfrostChainHistoryProvider({
client: blockfrostClient,
cache: createPersistentCacheStorage({
extensionLocalStorage,
fallbackMaxCollectionItemsGuard: cacheAssignment[CacheName.chainHistoryProvider].count,
resourceName: CacheName.chainHistoryProvider,
quotaInBytes: cacheAssignment[CacheName.chainHistoryProvider].size
}),
networkInfoProvider,
logger
});
const rewardsProvider = new BlockfrostRewardsProvider(blockfrostClient, logger);
const stakePoolProvider = stakePoolHttpProvider(httpProviderConfig);
const txSubmitProvider = createTxSubmitProvider(blockfrostClient, httpProviderConfig, customSubmitTxUrl);
Expand All @@ -122,7 +164,16 @@ export const createProviders = ({
stakePoolProvider
});

const inputResolver = new BlockfrostInputResolver(blockfrostClient, logger);
const inputResolver = new BlockfrostInputResolver({
cache: createPersistentCacheStorage({
extensionLocalStorage,
fallbackMaxCollectionItemsGuard: cacheAssignment[CacheName.inputResolver].count,
resourceName: CacheName.inputResolver,
quotaInBytes: cacheAssignment[CacheName.inputResolver].size
}),
client: blockfrostClient,
logger
});

if (useWebSocket) {
const url = new URL(baseUrl);
Expand Down Expand Up @@ -152,7 +203,16 @@ export const createProviders = ({
};
}

const utxoProvider = new BlockfrostUtxoProvider(blockfrostClient, logger);
const utxoProvider = new BlockfrostUtxoProvider({
cache: createPersistentCacheStorage({
extensionLocalStorage,
fallbackMaxCollectionItemsGuard: cacheAssignment[CacheName.utxoProvider].count,
resourceName: CacheName.utxoProvider,
quotaInBytes: cacheAssignment[CacheName.utxoProvider].size
}),
client: blockfrostClient,
logger
});

return {
assetProvider,
Expand Down
2 changes: 1 addition & 1 deletion packages/common/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
"watch": "yarn build --watch"
},
"dependencies": {
"@cardano-sdk/util": "0.15.6",
"@cardano-sdk/util": "0.15.7",
"antd": "^4.24.10",
"classnames": "^2.3.1",
"jdenticon": "3.1.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/core/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@
},
"dependencies": {
"@ant-design/icons": "^4.7.0",
"@cardano-sdk/wallet": "0.51.9",
"@cardano-sdk/web-extension": "0.38.10",
"@cardano-sdk/wallet": "0.51.10",
"@cardano-sdk/web-extension": "0.38.14",
"@input-output-hk/lace-ui-toolkit": "1.19.0",
"@lace/cardano": "0.1.0",
"@lace/common": "0.1.0",
Expand Down
10 changes: 5 additions & 5 deletions packages/nami/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@
},
"dependencies": {
"@biglup/is-cid": "^1.0.3",
"@cardano-sdk/core": "0.45.1",
"@cardano-sdk/crypto": "0.2.1",
"@cardano-sdk/tx-construction": "0.26.1",
"@cardano-sdk/util": "0.15.6",
"@cardano-sdk/web-extension": "0.38.10",
"@cardano-sdk/core": "0.45.2",
"@cardano-sdk/crypto": "0.2.2",
"@cardano-sdk/tx-construction": "0.26.2",
"@cardano-sdk/util": "0.15.7",
"@cardano-sdk/web-extension": "0.38.14",
"@chakra-ui/css-reset": "1.0.0",
"@chakra-ui/icons": "1.0.13",
"@chakra-ui/react": "1.6.4",
Expand Down
22 changes: 11 additions & 11 deletions packages/staking/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,12 @@
},
"devDependencies": {
"@babel/core": "^7.21.0",
"@cardano-sdk/core": "0.45.1",
"@cardano-sdk/input-selection": "0.14.2",
"@cardano-sdk/tx-construction": "0.26.1",
"@cardano-sdk/util": "0.15.6",
"@cardano-sdk/wallet": "0.51.9",
"@cardano-sdk/web-extension": "0.38.10",
"@cardano-sdk/core": "0.45.2",
"@cardano-sdk/input-selection": "0.14.3",
"@cardano-sdk/tx-construction": "0.26.2",
"@cardano-sdk/util": "0.15.7",
"@cardano-sdk/wallet": "0.51.10",
"@cardano-sdk/web-extension": "0.38.14",
"@storybook/addon-actions": "^7.6.7",
"@storybook/addon-essentials": "^7.6.7",
"@storybook/addon-interactions": "^7.6.7",
Expand Down Expand Up @@ -127,11 +127,11 @@
"wait-on": "^7.0.1"
},
"peerDependencies": {
"@cardano-sdk/input-selection": "0.14.2",
"@cardano-sdk/tx-construction": "0.26.1",
"@cardano-sdk/util": "0.15.6",
"@cardano-sdk/wallet": "0.51.9",
"@cardano-sdk/web-extension": "0.38.10",
"@cardano-sdk/input-selection": "0.14.3",
"@cardano-sdk/tx-construction": "0.26.2",
"@cardano-sdk/util": "0.15.7",
"@cardano-sdk/wallet": "0.51.10",
"@cardano-sdk/web-extension": "0.38.14",
"@lace/cardano": "^0.1.0",
"@lace/common": "^0.1.0",
"@lace/core": "0.1.0",
Expand Down
Loading
Loading