25:["$","title",null,{"children":"Filecoin Direct Data Onboarding (DDO) | Rod Vagg | Observable"}] 4b:T66c,chart = ({ data, categories, stack }) => { const width = 900 const height = 500 const margin = { top: 10, right: 0, bottom: 60, left: 80 } const numXTicks = 50 const xScale = d3.scaleBand( data.map((d) => d.day), [margin.left, width - margin.right] ).padding(0.2) const yMax = data.reduce((acc, d) => Math.max(categories.reduce((acc, cat) => Math.max(d[cat], acc), 0), acc), 0) const yScale = d3.scaleLinear( [0, yMax], [height - margin.bottom, margin.top] ) const xAxis = d3.axisBottom(xScale) .tickSizeOuter(0) .tickValues(xScale.domain().filter((_, i, array) => !(i % Math.ceil(array.length / numXTicks)))) const yAxis = d3.axisLeft(yScale) const svg = d3.select(DOM.svg(width, height)) .style('max-width', '100%') .style('height', 'auto') const chartData = stack(data) const groups = svg.append('g') .selectAll('g') .data(chartData) .join('g') .style('fill', (d, i) => colors(d.key.replace(/(GiB|Count)$/, ''))) groups.selectAll('rect') .data(d => d) .join('rect') .attr('x', d => xScale(d.data.day)) .attr('y', d => yScale(d[1])) .attr('height', d => yScale(d[0]) - yScale(d[1])) .attr('width', xScale.bandwidth()) svg.append('g') .attr('transform', `translate(0,${height - margin.bottom})`) .call(xAxis) .selectAll("text") .style("text-anchor", "end") .attr("dx", "-.8em") .attr("dy", ".15em") .attr("transform", "rotate(-65)"); svg.append('g') .attr('transform', `translate(${margin.left},0)`) .call(yAxis) .select('.domain') .remove() return svg.node() }4c:[] 47:{"id":"9449ea22a88d0602","slug":"filecoin-ddo-stats","title":"Filecoin Direct Data Onboarding (DDO)","thumbnail":"2fb36cf31a03740ff2ce58ecc29737371f849f74573726d097b5e6e72b96d466","access_level":"public","create_time":"$D2024-04-26T04:00:03.896Z","update_time":"$D2024-08-19T03:10:36.425Z","view_time":null,"first_public_time":"$D2024-04-26T09:15:45.389Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"bcd2457968b927c7","login":"rvagg","name":"Rod Vagg","avatar_url":"https://avatars-next.observableusercontent.com/a839d7bfbbc78c87811abb8b0be965e439e8d3b6937385fe4a92e2a699cfcb0e","tier":"starter_2024","bio":"Awk Ninja; Yak Shaving Rock Star","followed_by_you":false,"follows_you":false},"version":229,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"073315c2c7296ba0","login":"rvagg","name":"Rod Vagg","avatar_url":"https://avatars-next.observableusercontent.com/a839d7bfbbc78c87811abb8b0be965e439e8d3b6937385fe4a92e2a699cfcb0e","tier":null,"bio":"Awk Ninja; Yak Shaving Rock Star","followed_by_you":false,"follows_you":false}],"members":[{"type":"individual","id":"073315c2c7296ba0","login":"rvagg","name":"Rod Vagg","avatar_url":"https://avatars-next.observableusercontent.com/a839d7bfbbc78c87811abb8b0be965e439e8d3b6937385fe4a92e2a699cfcb0e","tier":null,"role":"editor"},{"type":"team","id":"bcd2457968b927c7","login":"rvagg","name":"Rod Vagg","avatar_url":"https://avatars-next.observableusercontent.com/a839d7bfbbc78c87811abb8b0be965e439e8d3b6937385fe4a92e2a699cfcb0e","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Filecoin Direct Data Onboarding (DDO)","cells":[{"id":29,"value":"# Filecoin Direct Data Onboarding (DDO)\n\nStorage onboarding patterns via `ProveCommitSectors3` and `ProveReplicaUpdates3`.\n\nThese miner actor methods are available for interacting with the market actor, and are therefore not exclusively a DDO pathway (where DDO means data onboarding not mediated by the built-in market actor). A breakdown is provided below, including details on non-market usage.\n\nSee [FIP-0076](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0076.md) for further details on these methods and how they enable DDO.\n\nData is collected with https://github.com/rvagg/spacemon","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":20,"value":"## All usage by total aggregate piece size\n\nAll `ProveCommitSectors3` and `ProveReplicaUpdates3` calls that contain pieces (i.e. are not Committed Capacity). This includes pieces that notify the market actor and may therefore be presumed to be related to a market deal.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":18,"value":"legend({ categories: gbCategories })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"chart({ data: ddoPerDay, categories: gbCategories, stack: stack(gbCategories) })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"## All usage by piece count","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":14,"value":"legend({ categories: countCategories })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"chart({ data: ddoPerDay, categories: countCategories, stack: stack(countCategories) })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"## Non-market usage by aggregate piece size\n\nExcluding pieces that notify the market actor. These pieces may be considered DDO. The unverified component is not recorded on chain, otherwise known as \"sparkling data\".","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":202,"value":"html`To date, there has been ${Number(Math.round(ddoData.filter((d) => d.pieces.some((p) => !p.f05)).reduce((acc, d) => acc + (d.pieces.reduce((acc, p) => acc + p.size, 0)) / 1024 / 1024 / 1024, 0) / 1024 / 1024 * 1000) / 1000).toLocaleString()} PiB total non-market DDO data onboarded`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"legend({ categories: nonMarketGbCategories })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"chart({ data: ddoPerDay, categories: nonMarketGbCategories, stack: stack(nonMarketGbCategories) })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"## Non-market usage by piece count","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":210,"value":"html`To date, there have been ${ddoData.filter((d) => d.pieces.some((p) => !p.f05)).reduce((acc, d) => acc + d.pieces.length, 0).toLocaleString()} total non-market DDO pieces onboarded`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"legend({ categories: nonMarketCountCategories })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"chart({ data: ddoPerDay, categories: nonMarketCountCategories, stack: stack(nonMarketCountCategories) })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":184,"value":"viewof miners = Inputs.button('List storage providers with non-market pieces', { value: [], reduce: () => {\n return ddoData\n .filter((d) => d.pieces.some((p) => !p.f05))\n .reduce((acc, d) => {\n if (!acc[d.provider]) {\n acc[d.provider] = {\n sectors: 0,\n pieces: 0,\n gib: 0\n }\n }\n acc[d.provider].sectors++\n acc[d.provider].pieces += d.pieces.length\n acc[d.provider].gib += (d.pieces.reduce((acc, p) => acc + p.size, 0)) / 1024 / 1024 / 1024\n return acc\n }, {})\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":188,"value":"html`

${provider} (${d.sectors} sectors, ${d.pieces} pieces, ${d.gib.toLocaleString()} GiB)`)}

${m.msg} (${m.pieceCount} pieces, ${m.time.toLocaleString().replace(/\\s/, ' ')})`)}

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":84,"value":"---\n## _Data and functions_","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":26,"value":"ddoData = {\n const ddoDataUrl = \"https://filvagg.s3.ap-southeast-2.amazonaws.com/compiled-ddo.json.gz\"\n return d3.json(ddoDataUrl)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":25,"value":"ddoPerDay = {\n const pieceType = (p) => `${p.verified ? 'verified' : 'unverified'}${p.f05 ? 'Market' : 'NonMarket'}`\n\n const categories = [\n pieceType({ verified: true, f05: true }),\n pieceType({ verified: true, f05: false }),\n pieceType({ verified: false, f05: true }),\n pieceType({ verified: false, f05: false })\n ]\n\n const initDay = (day) => {\n return categories.reduce((acc, c) => {\n acc[`${c}GiB`] = 0\n acc[`${c}Count`] = 0\n return acc\n }, { day: day })\n }\n\n const data = Object.values(ddoData.reduce((acc, e) => {\n const day = e.time.split('T')[0]\n if (acc[day] == null) {\n acc[day] = initDay(day)\n }\n e.pieces.forEach((p) => {\n const type = pieceType(p)\n acc[day][`${type}GiB`] += p.size / 1024 / 1024 / 1024\n acc[day][`${type}Count`]++\n })\n return acc\n }, {}))\n\n data.categories = categories\n\n return data\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":105,"value":"allCategories = ddoPerDay.categories.map((c) => `${c}GiB`).concat(ddoPerDay.categories.map((c) => `${c}Count`))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"gbCategories = ddoPerDay.categories.map((c) => `${c}GiB`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"countCategories = ddoPerDay.categories.map((c) => `${c}Count`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"nonMarketGbCategories = ddoPerDay.categories.filter((c) => /NonMarket/.test(c)).map((c) => `${c}GiB`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"nonMarketCountCategories = ddoPerDay.categories.filter((c) => /NonMarket/.test(c)).map((c) => `${c}Count`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":22,"value":"colors = d3.scaleOrdinal(ddoPerDay.categories, d3.schemeTableau10.slice(0))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"stack = (categories) => d3.stack().keys(categories)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":24,"value":"legend = ({ categories }) => {\n const width = 900\n const height = 25\n const paddingLeft = '7%'\n \n const svg = d3.select(DOM.svg(width, height))\n .style('max-width', '100%')\n .style('height', 'auto')\n .style('padding-left', paddingLeft)\n\n svg.selectAll('g')\n .data(categories.slice(0).reverse())\n .join('g')\n .attr('transform', (d, i) => `translate(${i * 200},5)`)\n .call(g => g\n .append('rect')\n .attr('width', 20)\n .attr('height', 20)\n .style('fill', d => colors(d.replace(/(GiB|Count)$/, '')))\n )\n .call(g => g\n .append('text')\n .attr('y', 10)\n .attr('x', 25)\n .attr('dy', '0.4em')\n .style('font-size', 12)\n .style('font-family', 'sans-serif')\n .text((d) => {\n return d[0].toUpperCase() + d.match(/(^|GiB|[A-Z])[a-z]*/g).join(' ').slice(1).replace(/Non M/, 'Non-m')\n })\n )\n\n return svg.node()\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"$4b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4c","stdlib":"1"}} 24:null