diff --git a/.gitignore b/.gitignore index f24507fb..e8320197 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ yarn-error.log .DS_Store .idea/ .vscode/ + +src/edgeRateIdUtils/data \ No newline at end of file diff --git a/package.json b/package.json index 13f64942..714c8e99 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "@typescript-eslint/eslint-plugin": ">=2.0.0", "@typescript-eslint/parser": "^2.0.0", "chai": "^4.2.0", + "disklet": "^0.5.2", "eslint": ">=6.2.2", "eslint-config-standard-kit": ">=0.14.4", "eslint-plugin-import": ">=2.18.0", diff --git a/src/edgeRateIdUtils/coingecko.ts b/src/edgeRateIdUtils/coingecko.ts new file mode 100644 index 00000000..fba83c3e --- /dev/null +++ b/src/edgeRateIdUtils/coingecko.ts @@ -0,0 +1,228 @@ +import { asArray, asObject, asString } from 'cleaners' + +import { config } from '../config' +import { + contractAddressToTokenId, + CreateTokenId, + EdgePluginIdPlatformMap, + EdgeTokenIdUniqueIdMap +} from './tokenIdUtils' + +const asCoinGeckoAssets = asArray( + asObject({ + id: asString, + symbol: asString, + name: asString, + platforms: asObject(asString) + }) +) + +export const coingecko = async (): Promise => { + const { apiKey } = config.providers.coingeckopro + + const res = await fetch( + 'https://pro-api.coingecko.com/api/v3/coins/list?include_platform=true', + { headers: { 'x-cg-pro-api-key': apiKey } } + ) + const json = await res.json() + const assets = asCoinGeckoAssets(json) + + const coingeckoPluginIdMap: { [key: string]: string } = {} + for (const [key, value] of Object.entries(edgePluginIdToCoingeckoId)) { + if (value != null) coingeckoPluginIdMap[value] = key + } + const coingeckoPlatformPluginIdMap: { [key: string]: string } = {} + for (const [key, value] of Object.entries( + edgePluginIdToCoingeckoIdPlatform + )) { + if (value != null) coingeckoPlatformPluginIdMap[value] = key + } + + const out: EdgeTokenIdUniqueIdMap = {} + for (const asset of assets) { + if (asset.id in coingeckoPluginIdMap) { + const edgeRateId = `${coingeckoPluginIdMap[asset.id]}_null` + out[edgeRateId] = asset.id + } else { + for (const [platform, contractAddress] of Object.entries( + asset.platforms + )) { + const parentPluginId = coingeckoPlatformPluginIdMap[platform] + if (parentPluginId == null) continue + + const toTokenId: CreateTokenId | null = + contractAddressToTokenId[parentPluginId] + if (toTokenId == null) continue + + const tokenEdgeRateId = `${parentPluginId}_${toTokenId( + contractAddress + )}` + out[tokenEdgeRateId] = asset.id + } + } + } + + return out +} + +export const edgePluginIdToCoingeckoIdPlatform: EdgePluginIdPlatformMap = { + // edge-currency-accountbased: + amoy: null, // Polygontestnet + arbitrum: 'arbitrum-one', + algorand: 'algorand', + avalanche: 'avalanche', + axelar: null, + base: 'base', + binance: 'binancecoin', + binancesmartchain: 'binance-smart-chain', + bobevm: 'bob-network', + cardano: 'cardano', + cardanotestnet: null, // CardanoTestnet + celo: 'celo', + coreum: null, + cosmoshub: 'cosmos', + eos: 'eos', + ethereum: 'ethereum', + ethereumclassic: 'ethereum-classic', + ethereumpow: 'ethereumpow', + fantom: 'fantom', + filecoin: null, + filecoinfevm: 'filecoin', + filecoinfevmcalibration: null, // FilecoinEVMtestnet + fio: null, + hedera: 'hedera-hashgraph', + holesky: null, // EthereumTestnet + liberland: null, + liberlandtestnet: null, // Liberland testnet + optimism: 'optimistic-ethereum', + osmosis: 'osmosis', + piratechain: null, + polkadot: null, + polygon: 'polygon-pos', + pulsechain: 'pulsechain', + ripple: 'xrp', + rsk: 'rootstock', + sepolia: null, // EthereumTestnet + solana: 'solana', + stellar: 'stellar', + telos: 'telos', + tezos: 'tezos', + thorchainrune: null, + tron: 'tron', + wax: 'wax', + zksync: 'zksync', + zcash: null, + // edge-currency-bitcoin: + bitcoin: null, + bitcoincash: null, + bitcoincashtestnet: null, + bitcoingold: null, + bitcoingoldtestnet: null, + bitcoinsv: null, + bitcointestnet: null, + dash: null, + digibyte: null, + dogecoin: null, + eboost: null, + feathercoin: null, + groestlcoin: null, + litecoin: null, + qtum: null, + ravencoin: null, + smartcash: null, + ufo: null, + vertcoin: null, + zcoin: null, + // edge-currency-monero: + monero: null +} + +// const getPlatformIds = ( +// data: ReturnType +// ): string[] => { +// const out = new Set() +// for (const asset of data) { +// for (const platform of Object.keys(asset.platforms)) { +// out.add(platform) +// } +// } +// return [...out] +// } + +export const edgePluginIdToCoingeckoId: EdgePluginIdPlatformMap = { + // edge-currency-accountbased: + amoy: null, // Polygontestnet + arbitrum: 'arbitrum-one', + algorand: 'algorand', + avalanche: 'avalanche', + axelar: 'axelar', + base: 'base', + binance: 'binancecoin', + binancesmartchain: 'binance-smart-chain', + bobevm: 'bob-network', // TODO: + cardano: 'cardano', + cardanotestnet: null, // CardanoTestnet + celo: 'celo', + coreum: 'coreum', + cosmoshub: 'cosmos', + eos: 'eos', + ethereum: 'ethereum', + ethereumclassic: 'ethereum-classic', + ethereumpow: 'ethereumpow', + fantom: 'fantom', + filecoin: null, + filecoinfevm: 'filecoin', + filecoinfevmcalibration: null, // FilecoinEVMtestnet + fio: 'fio-protocol', + hedera: 'hedera-hashgraph', + holesky: null, // EthereumTestnet + liberland: 'liberland-lld', + liberlandtestnet: null, // Liberland testnet + optimism: 'optimistic-ethereum', + osmosis: 'osmosis', + piratechain: 'pirate-chain', + polkadot: 'polkadot', + polygon: 'matic-network', + pulsechain: 'pulsechain', + ripple: 'ripple', + rsk: 'rootstock', + sepolia: null, // EthereumTestnet + solana: 'solana', + stellar: 'stellar', + telos: 'telos', + tezos: 'tezos', + thorchainrune: 'thorchain', + tron: 'tron', + wax: 'wax', + zksync: 'zksync', + zcash: 'zcash', + // edge-currency-bitcoin: + bitcoin: 'bitcoin', + bitcoincash: 'bitcoin-cash', + bitcoincashtestnet: null, + bitcoingold: 'bitcoin-gold', + bitcoingoldtestnet: null, + bitcoinsv: 'bitcoin-cash-sv', + bitcointestnet: null, + dash: 'dash', + digibyte: 'digibyte', + dogecoin: 'dogecoin', + eboost: 'eboost', + feathercoin: 'feathercoin', + groestlcoin: 'groestlcoin', + litecoin: 'litecoin', + qtum: 'qtum', + ravencoin: 'ravencoin', + smartcash: 'smartcash', + ufo: 'ufocoin', + vertcoin: 'vertcoin', + zcoin: 'zcoin', + // edge-currency-monero: + monero: 'monero' +} + +if (process.argv[1].includes('coingecko.ts')) { + coingecko() + .then(data => console.log(data)) + .catch(console.error) +} diff --git a/src/edgeRateIdUtils/coinmarketcap.ts b/src/edgeRateIdUtils/coinmarketcap.ts new file mode 100644 index 00000000..bc4c588d --- /dev/null +++ b/src/edgeRateIdUtils/coinmarketcap.ts @@ -0,0 +1,265 @@ +import { + asArray, + asEither, + asMaybe, + asNull, + asNumber, + asObject, + asString, + asUnknown +} from 'cleaners' +import { join } from 'path' + +import { makeNodeDisklet } from '../../node_modules/disklet' +import { config } from '../config' +import { snooze } from '../utils/utils' +import { + contractAddressToTokenId, + CreateTokenId, + EdgePluginIdPlatformMap, + EdgeTokenIdUniqueIdMap +} from './tokenIdUtils' + +const asCoinmarketcapAsset = asObject({ + id: asNumber, + name: asString, + symbol: asString, + // slug: asString, + // is_active: asMaybe(asNumber), + + platform: asEither( + asNull, + asObject({ + id: asNumber, + name: asString, + symbol: asString, + // slug: asString, + // eslint-disable-next-line @typescript-eslint/camelcase + token_address: asString + }) + ) +}) + +const asCoinmarketcapRes = asObject({ + status: asUnknown, + data: asArray(asCoinmarketcapAsset) +}) + +const asCoinmarketcapMetadata = asObject( + asObject({ + id: asNumber, + name: asString, + symbol: asString, + // platform: asObject({ + // id: asString, + // name: asString, + // slug: asString, + // symbol: asString, + // // eslint-disable-next-line @typescript-eslint/camelcase + // token_address: asString + // }), + // eslint-disable-next-line @typescript-eslint/camelcase + contract_address: asArray( + asObject({ + // eslint-disable-next-line @typescript-eslint/camelcase + contract_address: asString, + platform: asObject({ + name: asString, + coin: asObject({ + id: asString, + name: asString, + symbol: asString, + slug: asString + }) + }) + }) + ) + }) +) + +const asCoinmarketcapMetadataRes = asObject({ + status: asUnknown, + data: asCoinmarketcapMetadata +}) + +export const coinmarketcap = async (): Promise => { + const disklet = makeNodeDisklet(join(__dirname, './')) + const files = await disklet.list('./data') + if (files['data/coinmarketcap.json'] == null) { + await disklet.setText('./data/coinmarketcap.json', '{}') + } + const savedJson = JSON.parse( + await disklet.getText('./data/coinmarketcap.json') + ) + + const { apiKey } = config.providers.coinMarketCapHistorical + + const numberOfAssets = Object.keys(savedJson).length + const ITEMS_PER_PAGE = 5000 + let page = Math.floor(numberOfAssets / ITEMS_PER_PAGE) + let foundNew = false + while (true) { + const res = await fetch( + `https://pro-api.coinmarketcap.com/v1/cryptocurrency/map?listing_status=active,inactive,untracked&start=${page * + ITEMS_PER_PAGE + + 1}`, + { headers: { 'X-CMC_PRO_API_KEY': apiKey } } + ) + const json = await res.json() + const clean = asCoinmarketcapRes(json) + + for (const asset of clean.data) { + if (savedJson[asset.id] == null) { + foundNew = true + savedJson[asset.id] = asset + } + } + if (clean.data.length < ITEMS_PER_PAGE) { + break + } + page++ + await snooze(1000) + } + + if (foundNew) { + await disklet.setText( + './data/coinmarketcap.json', + JSON.stringify(savedJson) + ) + } + + const coinmarketcapPluginIdMap: { [key: string]: string } = {} + for (const [key, value] of Object.entries(edgePluginIdToCoinmarketcapId)) { + if (value != null) coinmarketcapPluginIdMap[value] = key + } + + const out: EdgeTokenIdUniqueIdMap = {} + + const metadataNeededIds: number[] = [] + for (const asset of Object.values(savedJson)) { + const cleanAsset = asMaybe(asCoinmarketcapAsset)(asset) + if (cleanAsset == null) continue + if (cleanAsset.platform !== null) { + metadataNeededIds.push(cleanAsset.id) + } else { + const parentPluginId = coinmarketcapPluginIdMap[cleanAsset.id] + if (parentPluginId == null) continue + + const parentEdgeRateId = `${parentPluginId}_null` + out[parentEdgeRateId] = cleanAsset.id.toString() + } + } + + const METADATA_LIMIT_PER_PAGE = 100 + for (let i = 0; i < metadataNeededIds.length; i += METADATA_LIMIT_PER_PAGE) { + if (i > 500) break + const chunk = metadataNeededIds.slice(i, i + METADATA_LIMIT_PER_PAGE) + const res = await fetch( + `https://pro-api.coinmarketcap.com/v2/cryptocurrency/info?id=${chunk.join( + ',' + )}&aux=platform`, + { headers: { 'X-CMC_PRO_API_KEY': apiKey } } + ) + const json = await res.json() + const clean = asCoinmarketcapMetadataRes(json) + + for (const asset of Object.values(clean.data)) { + const { contract_address: contractAddresses, id } = asset + for (const contractAddress of contractAddresses) { + const { contract_address: address, platform } = contractAddress + const parentPluginId = coinmarketcapPluginIdMap[platform.coin.id] + if (parentPluginId == null) continue + const parentEdgeRateId = `${parentPluginId}_null` + if (out[parentEdgeRateId] == null) { + out[parentEdgeRateId] = platform.coin.id + } + + const toTokenId: CreateTokenId | null = + contractAddressToTokenId[parentPluginId] + if (toTokenId == null) continue + const tokenEdgeRateId = `${parentPluginId}_${toTokenId(address)}` + out[tokenEdgeRateId] = id.toString() + } + } + } + + return out +} + +export const edgePluginIdToCoinmarketcapId: EdgePluginIdPlatformMap = { + // edge-currency-accountbased: + amoy: null, // Polygon testnet + arbitrum: '11841', + algorand: '4030', + avalanche: '5805', + axelar: '17799', + base: '7838', + binance: null, + binancesmartchain: '1839', + bobevm: null, + cardano: '2010', + cardanotestnet: null, // Cardano Testnet + celo: '5567', + coreum: '16399', + cosmoshub: '3794', + eos: '1765', + ethereum: '1027', + ethereumclassic: '1321', + ethereumpow: '21296', + fantom: '3513', + filecoin: null, + filecoinfevm: '2280', + filecoinfevmcalibration: null, // FilecoinEVM testnet + fio: '5865', + hedera: '4642', + holesky: null, // Ethereum Testnet + liberland: null, + liberlandtestnet: null, // Liberland testnet + optimism: '11840', + osmosis: '12220', + piratechain: '3951', + polkadot: '6636', + polygon: '3890', + pulsechain: '28928', + ripple: '52', + rsk: '3626', + sepolia: null, // Ethereum Testnet + solana: '5426', + stellar: '512', + telos: '4660', + tezos: '2011', + thorchainrune: '4157', + tron: '1958', + wax: '2300', + zksync: '24091', + zcash: '1437', + // edge-currency-bitcoin: + bitcoin: '1', + bitcoincash: '1831', + bitcoincashtestnet: null, + bitcoingold: '2083', + bitcoingoldtestnet: null, + bitcoinsv: '3602', + bitcointestnet: null, + dash: '131', + digibyte: '109', + dogecoin: '74', + eboost: '1704', + feathercoin: '8', + groestlcoin: '258', + litecoin: '2', + qtum: '1684', + ravencoin: '2577', + smartcash: '1828', + ufo: '168', + vertcoin: '99', + zcoin: '1414', + // edge-currency-monero: + monero: '328' +} + +if (process.argv[1].includes('coinmarketcap.ts')) { + coinmarketcap() + .then(data => console.log(data)) + .catch(console.error) +} diff --git a/src/edgeRateIdUtils/tokenIdUtils.ts b/src/edgeRateIdUtils/tokenIdUtils.ts new file mode 100644 index 00000000..dd14f2ce --- /dev/null +++ b/src/edgeRateIdUtils/tokenIdUtils.ts @@ -0,0 +1,92 @@ +export type CreateTokenId = (contractAddress: string) => string + +const toEvmTokenId: CreateTokenId = (contractAddress: string): string => + contractAddress.replace('0x', '').toLowerCase() +const toCosmosTokenId: CreateTokenId = (contractAddress: string): string => + contractAddress.toLowerCase().replace(/\//g, '') +const toEosTokenId: CreateTokenId = (contractAddress: string): string => + contractAddress.toLowerCase() +const toDefaultTokenId: CreateTokenId = (contractAddress: string): string => + contractAddress // Use the contract address as-is + +export const contractAddressToTokenId = { + // edge-currency-accountbased: + algorand: toDefaultTokenId, + amoy: toEvmTokenId, // Polygon testnet + arbitrum: toEvmTokenId, + avalanche: toEvmTokenId, + axelar: toCosmosTokenId, + base: toEvmTokenId, + binance: null, + binancesmartchain: toEvmTokenId, + bobevm: toEvmTokenId, + cardano: null, + cardanotestnet: null, // Cardano Testnet + celo: toEvmTokenId, + coreum: toCosmosTokenId, + cosmoshub: toCosmosTokenId, + eos: toEosTokenId, + ethereum: toEvmTokenId, + ethereumclassic: toEvmTokenId, + ethereumpow: toEvmTokenId, + fantom: toEvmTokenId, + filecoin: null, + filecoinfevm: toEvmTokenId, + filecoinfevmcalibration: toEvmTokenId, // FilecoinEVM testnet + fio: null, + hedera: null, + holesky: toEvmTokenId, // Ethereum Testnet + liberland: null, + liberlandtestnet: null, // Liberland testnet + optimism: toEvmTokenId, + osmosis: toCosmosTokenId, + piratechain: null, + polkadot: toDefaultTokenId, + polygon: toEvmTokenId, + pulsechain: toEvmTokenId, + ripple: null, + rsk: toEvmTokenId, + sepolia: toEvmTokenId, // Ethereum Testnet + solana: toDefaultTokenId, + stellar: null, + telos: toEosTokenId, + tezos: null, + thorchainrune: toCosmosTokenId, + tron: toDefaultTokenId, + wax: toEosTokenId, + zcash: null, + zksync: toEvmTokenId, + // edge-currency-bitcoin: + bitcoin: null, + bitcoincash: null, + bitcoincashtestnet: null, + bitcoingold: null, + bitcoingoldtestnet: null, + bitcoinsv: null, + bitcointestnet: null, + dash: null, + digibyte: null, + dogecoin: null, + eboost: null, + feathercoin: null, + groestlcoin: null, + litecoin: null, + qtum: null, + ravencoin: null, + smartcash: null, + ufo: null, + vertcoin: null, + zcoin: null, + // edge-currency-monero: + monero: null +} + +export type EdgePluginIds = keyof typeof contractAddressToTokenId + +export type EdgePluginIdPlatformMap = { + [key in EdgePluginIds]: string | null +} + +export interface EdgeTokenIdUniqueIdMap { + [key: string]: string +} diff --git a/yarn.lock b/yarn.lock index 8a89bdd1..6c803eeb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1015,6 +1015,13 @@ diff@3.5.0: resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== +disklet@^0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/disklet/-/disklet-0.5.2.tgz#eb1b3bfc2840883cb432aaa16d2c78a345cdd778" + integrity sha512-Fx9LFHztHa47QVCHKaABvk+R/zInmi17HweWM+PQyO3bv55E709IPcJIMOH1WyuPNTPHAAM9GToN6NgpdLhUCQ== + dependencies: + rfc4648 "^1.3.0" + doctrine@1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-1.5.0.tgz#379dce730f6166f76cefa4e6707a159b02c5a6fa" @@ -2955,6 +2962,11 @@ restore-cursor@^3.1.0: onetime "^5.1.0" signal-exit "^3.0.2" +rfc4648@^1.3.0: + version "1.5.3" + resolved "https://registry.yarnpkg.com/rfc4648/-/rfc4648-1.5.3.tgz#e62b81736c10361ca614efe618a566e93d0b41c0" + integrity sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ== + rimraf@2.6.3: version "2.6.3" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab"