Published
Edited
May 25, 2022
allFundedWithMinerId = (await fetch(minersAndFundersUrl)).json()
roots = allFundedWithMinerId.filter(({ funded_from }) => !funded_from).sort(sortIdRecords)
Inputs.table(roots, { columns: [ "id", "address" ] })
joinedAllFundedWithMinerId = allFundedWithMinerId
.map(record => ({
...record,
funded_from: record.funded_from || 'root'
}))
.concat([{
id: "root",
address: "root",
funded_from: null,
miner_id: null
}])
minersWithFundingDataSet = {
const minersSet = new Set()
for (const { miner_id } of joinedAllFundedWithMinerId) {
minersSet.add(miner_id)
}
return minersSet
}
minersWithPowerSet = new Set(Object.keys(minerPowerLatestReport.miners))
minersWithNoFundingSet = {
const minersSet = new Set()
for (const minerId of [...minersWithPowerSet]) {
if (!minersWithFundingDataSet.has(minerId)) {
minersSet.add(minerId)
}
}
return minersSet
}
fakeFundingRecords = [...minersWithNoFundingSet].map(minerId => ({
id: minerId,
address: minerId,
funded_from: "root",
miner_id: minerId
}))
joinedAllFundedWithMinerIdAndFakes = {
return joinedAllFundedWithMinerId
// FIXME: Joining records at the root appears to have over-allocated delegates to Vietnam - investigate
// joinedAllFundedWithMinerId.concat(fakeFundingRecords)
}
reachable = {
const addressIndex = d3.index(joinedAllFundedWithMinerIdAndFakes, d => d.id)
const fundedFromIndex = d3.group(joinedAllFundedWithMinerIdAndFakes, d => d.funded_from)
const start = addressIndex.get('root')
const reachable = []
function walk (from) {
reachable.push(from)
// console.log('Walking', from.id)
const fundedNodes = fundedFromIndex.get(from.address)
// console.log('Funded Nodes', fundedNodes)
if (fundedNodes) {
for (const funded of fundedNodes) {
walk(funded)
}
}
}
walk(start)
return reachable
}
stratify = d3.stratify()
.id(d => d["address"])
.parentId(d => d["funded_from"])
reachableTree = stratify(reachable)
/* Too big!! graph(reachableTree, {label: d => d.data.id + (d.data.miner_id ? ` (SP: ${d.data.miner_id})` : '') }) */
leavesWithPower = reachable.filter(({ miner_id }) => minerPowerLatestReport.miners[miner_id])
reachableWithPower = {
const addressIndex = d3.index(reachable, d => d.address)
const filtered = new Map()
function walkUp (node) {
// console.log('WalkUp', node)
filtered.set(node.id, {
...node,
qualityAdjPower: minerPowerDailyAverageReport.miners[node.miner_id] &&
minerPowerDailyAverageReport.miners[node.miner_id].qualityAdjPower
})
if (node.funded_from) {
// console.log('funded_from', node.funded_from, addressIndex)
walkUp(addressIndex.get(node.funded_from))
}
}
for (const leaf of leavesWithPower) {
walkUp(leaf)
}
return [...filtered.values()].sort(sortIdRecords)
}
reachableWithPowerTree = stratify(reachableWithPower)
shortCircuit1 = true
!shortCircuit1 && graph(reachableWithPowerTree, {label: d => d.data.id + (d.data.miner_id ? ` (SP: ${d.data.miner_id} - ${bytes(d.data.qualityAdjPower, { mode: 'binary' })})` : '') })
reachableWithPowerAndRegions = reachableWithPower.map(record => {
const newRecord = record
if (record.miner_id) {
const regions = minerRegionsCSPReport.minerRegions.filter(({ miner }) => miner === record.miner_id)
if (regions.length > 0) {
newRecord.regions = regions.map(({ region }) => region)
}
}
return newRecord
})
button(reachableWithPowerAndRegions, 'funder-tree-base.json')
reachableWithPowerAndRegionsTree = stratify(reachableWithPowerAndRegions)
shortCircuit2 = true
!shortCircuit2 && graph(reachableWithPowerAndRegionsTree, {
label: d =>
(d.data.miner_id ?
`SP: ${d.data.miner_id} - ${bytes(d.data.qualityAdjPower, { mode: 'binary' })}` : d.data.id) +
(d.data.regions ?
` - ${d.data.regions.join(', ')}` : '')
})
async function *getTreeWithDelegatesStream (tree) {
const providerLeaves = []
for (const provider of tree.leaves()) {
if (!provider.data.regions) {
providerLeaves.push(provider)
}
}
let count = 0
for (const provider of providerLeaves) {
console.log(`Computing delegate for ${provider.data.id}`)
const delegateId = await matchDelegate(provider)
// const delegateId = 'xxx'
console.log(`Delegate: ${provider.data.id} => ${delegateId}`)
provider.data.delegateId = delegateId
yield {
done: false,
processed: ++count,
total: providerLeaves.length
}
}
yield {
done: true,
tree
}
}
viewof start = Inputs.button('Start')
funderTreeWithDelegatesProgress = start && getTreeWithDelegatesStream(reachableWithPowerAndRegionsTree)
funderTreeWithDelegatesProgressWithoutResult = {
if (!funderTreeWithDelegatesProgress) return false
const { tree, ...rest } = funderTreeWithDelegatesProgress
return { ...rest }
}
funderTreeWithDelegates = funderTreeWithDelegatesProgress && funderTreeWithDelegatesProgress.done ? funderTreeWithDelegatesProgress.tree : null
Synthetic Provider Regions
providerCSPRegions = d3.group(minerRegionsCSPReport.minerRegions, d => d.miner)
syntheticProviderCSPRegions = {
if (!funderTreeWithDelegates) return
const providerRegionsWithDelegates = []
for (const provider of funderTreeWithDelegates.leaves()) {
const { miner_id: providerId, delegateId } = provider.data
const targetId = delegateId || providerId
const targetRegions = providerCSPRegions.get(targetId)
if (targetRegions) {
for (const region of targetRegions) {
const outputRegion = {
provider: providerId,
region: region.region,
numRegions: region.numRegions
}
if (delegateId) {
outputRegion.delegate = delegateId
}
providerRegionsWithDelegates.push(outputRegion)
}
}
}
return providerRegionsWithDelegates.sort(sortProviderRecords)
}
syntheticProviderCSPRegions && button(syntheticProviderCSPRegions, `synthetic-provider-country-state-province-${minerRegionsCSPReport.epoch}.json`)
providerRegions = d3.group(minerRegionsReport.minerRegions, d => d.miner)
syntheticProviderRegions = {
if (!funderTreeWithDelegates) return
const providerRegionsWithDelegates = []
for (const provider of funderTreeWithDelegates.leaves()) {
const { miner_id: providerId, delegateId } = provider.data
const targetId = delegateId || providerId
const targetRegions = providerRegions.get(targetId)
if (targetRegions) {
for (const region of targetRegions) {
const outputRegion = {
provider: providerId,
region: region.region,
numRegions: region.numRegions
}
if (delegateId) {
outputRegion.delegate = delegateId
}
providerRegionsWithDelegates.push(outputRegion)
}
}
}
return providerRegionsWithDelegates.sort(sortProviderRecords)
}
syntheticProviderRegions && button(syntheticProviderRegions, `synthetic-provider-regions-${minerRegionsReport.epoch}.json`)
Synthetic Provider Locations
providerCSPLocations = d3.group(minerLocationsCSPReport.minerLocations, d => d.miner)
syntheticProviderCSPLocations = {
if (!funderTreeWithDelegates) return
const providerLocationsWithDelegates = []
for (const provider of funderTreeWithDelegates.leaves()) {
const { miner_id: providerId, delegateId } = provider.data
const targetId = delegateId || providerId
const targetLocations = providerCSPLocations.get(targetId)
if (targetLocations) {
for (const location of targetLocations) {
const { miner, ...rest } = location
const outputLocation = {
provider: providerId,
...rest
}
if (delegateId) {
outputLocation.delegate = delegateId
}
providerLocationsWithDelegates.push(outputLocation)
}
}
}
return providerLocationsWithDelegates.sort(sortProviderRecords)
}
syntheticProviderCSPLocations && button(syntheticProviderCSPLocations, `synthetic-provider-country-state-province-locations-${minerLocationsCSPReport.epoch}.json`)
providerLocations = d3.group(minerLocationsReport.minerLocations, d => d.miner)
syntheticProviderLocations = {
if (!funderTreeWithDelegates) return
const providerLocationsWithDelegates = []
for (const provider of funderTreeWithDelegates.leaves()) {
const { miner_id: providerId, delegateId } = provider.data
const targetId = delegateId || providerId
const targetLocations = providerLocations.get(targetId)
if (targetLocations) {
for (const location of targetLocations) {
const { miner, ...rest } = location
const outputLocation = {
provider: providerId,
...rest
}
if (delegateId) {
outputLocation.delegate = delegateId
}
providerLocationsWithDelegates.push(outputLocation)
}
}
}
return providerLocationsWithDelegates.sort(sortProviderRecords)
}
syntheticProviderLocations && button(syntheticProviderLocations, `synthetic-provider-locations-${minerLocationsReport.epoch}.json`)
minerPowerDailyAverageReport = (await fetch(`${minerPowerDailyAverageLatestBucketUrl}/miner-power-daily-average-latest.json`)).json()
minerPowerLatestReport = (await fetch(`${minerPowerDailyAverageLatestBucketUrl}/miner-power-latest.json`)).json()
geoIpLookupsBucketUrl
minerRegionsReport = (await fetch(`${geoIpLookupsBucketUrl}/miner-regions-latest.json`)).json()
minerRegionsCSPReport = (await fetch(`${geoIpLookupsBucketUrl}/provider-country-state-province-latest.json`)).json()
minerLocationsReport = (await fetch(`${geoIpLookupsBucketUrl}/miner-locations-latest.json`)).json()
minerLocationsCSPReport = (await fetch(`${geoIpLookupsBucketUrl}/provider-country-state-province-locations-latest.json`)).json()
sortIdRecords = ({ id: minerA }, { id: minerB }) => Number(minerA.slice(1)) - Number(minerB.slice(1))
sortProviderRecords = ({ provider: providerA }, { provider: providerB }) => Number(providerA.slice(1)) - Number(providerB.slice(1))
import {dateToEpoch, epochToDate} from '@jbenet/filecoin-chain-time-calculator'
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.