2b:null 2c:["$","title",null,{"children":"Hypercore Data + Tree in IPLD | Jim Pick | Observable"}] 31:T4af,{ for await (const treeData of onTreeIterator()) { await Promises.delay(100) const nodes = getNodes(feed.length) const printTree = printFlatTree(nodes).split('\n') const lines = [] let index = 0 for (let i = 0; i < printTree.length - 1; i++) { let dataIndex = '' let data = '' if (i % 2 == 0) { dataIndex = `${i / 2}` data = String(await feedGet(feed, i / 2)) } lines.push( dataIndex.padEnd(3) + data.padEnd(12) + printTree[i] ) } const width = lines.reduce((acc, line) => Math.max(acc, line.length), 0) const lines2 = [] for (let i = 0; i < printTree.length - 1; i++) { let dataIndex = '' let size = '' let hash = '' if (i % 2 == 0) { dataIndex = `${i / 2}` // size = 'Size: ' + String(await feedGet(feed, i / 2)).length } if (treeData[i]) { size = 'Size: ' + treeData[i].size hash = 'Hash: ' + treeData[i].hash } lines2.push( lines[i].padEnd(width + 1) + size.padEnd(10) + hash ) } yield html`

${lines2.join('\n')}

` } }32:[] 2f:{"id":"fc9c61946d10a444","slug":"hypercore-data-tree-in-ipld","title":"Hypercore Data + Tree in IPLD","thumbnail":"29d2e824ded8f9fc346ab336d5c33e4b4e8e74fc0fa5641aa2836058e86c16ab","access_level":"public","create_time":"$D2020-07-31T14:29:58.716Z","update_time":"$D2020-07-31T15:17:43.864Z","view_time":null,"first_public_time":"$D2020-07-31T15:08:26.579Z","fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"5252cea433451319","login":"jimpick","name":"Jim Pick","avatar_url":"https://avatars-next.observableusercontent.com/e8813527082139d5411445010e72fd85f99b04a26355e795d0ea26765425d0cd","tier":"starter_2024"},"version":94,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"bda4505ba9a55ac8","login":"jimpick","name":"Jim Pick","avatar_url":"https://avatars-next.observableusercontent.com/e8813527082139d5411445010e72fd85f99b04a26355e795d0ea26765425d0cd","tier":null}],"members":[{"type":"individual","id":"bda4505ba9a55ac8","login":"jimpick","name":"Jim Pick","avatar_url":"https://avatars-next.observableusercontent.com/e8813527082139d5411445010e72fd85f99b04a26355e795d0ea26765425d0cd","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":2,"collection":{"id":"c273e1e644de46b3","slug":"hypercore-ipld","title":"hypercore-ipld","owner":{"id":"5252cea433451319","login":"jimpick","name":"Jim Pick"}},"type":"notebook","body":{"title":"Hypercore Data + Tree in IPLD","cells":[{"id":0,"value":"md`# Hypercore Data + Tree in IPLD\n\nThere are many possible ways to represent the data in IPLD. If you recall, Hypercore has a \\`data\\` SLEEP file that holds the raw binary data for the append-only log records, and a \\`tree\\` SLEEP file that holds metadata about those records (size and hash).\n\nBut IPLD is talented at representing graph datastructures. We could just build a graph that matches the Hypercore binary merkel tree, and attach the binary data directly as links. This is very similar to the \\`UnixFS\\` data structure that IPFS uses to store directories of files. Let's try that!`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":51,"value":"md`# Hypercore review\n\nLet's use our previous Hypercore example.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"viewof b1 = appendButton()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":37,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":55,"value":"md`## Data leaves\n\nLet's iterate over the leaves and make them into IPLD data with CIDs.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":67,"value":"ipfsNode = await ipfs.create()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":59,"value":"{\n for await (const treeData of onTreeIterator()) {\n await Promises.delay(100)\n const dataCids = []\n for (let i = 0; i < feed.length; i++) {\n const options = { format: 'raw', hashAlg: 'blake2b-256' }\n const data = await feedGet(feed, i)\n const leaf = Buffer.concat([\n Buffer.from([0]),\n uint64be.encode(data.length),\n data\n ])\n const cid = await ipfsNode.dag.put(leaf, options)\n dataCids.push(cid.toString())\n }\n yield dataCids\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":83,"value":"md`## Tree nodes\n\n... to be continued\n\n[GitHub prototype](https://github.com/jimpick/hypercore-simple-ipld/blob/master/dump-to-ipfs.js#L78)\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":86,"value":"md`## Root nodes`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":90,"value":"md`## Other SLEEP files\n\n* signatures\n* bitfields\n* keys\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":92,"value":"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"md`## Imports and demo helpers`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"import {hypercore, ram, onAppendAsyncIterator, feedGet, appendRandomFruit} from '@jimpick/hypercore-intro'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"randomAccess = require('https://bundle.run/random-access-storage@1.4.1')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":39,"value":"printFlatTree = require('https://bundle.run/print-flat-tree@2.2.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"uint64be = require('https://bundle.run/uint64be@3.0.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":18,"value":"import {shimAndIterators, getNodes} from '@jimpick/inside-hypercore'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"storageShim = shimAndIterators.storageShim","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":26,"value":"onDataIterator = shimAndIterators.onDataIterator","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":29,"value":"onTreeIterator = shimAndIterators.onTreeIterator","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":33,"value":"feed = hypercore(storageShim)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":35,"value":"{\n await Promises.delay(1000)\n appendRandomFruit(feed);\n return \"init with random fruit\"\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":49,"value":"function appendButton () {\n const div = html`\n

\n \n \n

\n `\n const button = div.querySelector('button')\n const span = div.querySelector('span')\n \n button.onclick = () => {\n const fruit = appendRandomFruit(feed)\n span.innerText = `Appended: ${fruit}`\n }\n return div\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":65,"value":"ipfs = require('ipfs@0.48.1/dist/index.min.js')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"buf = require('https://bundle.run/buffer@5.6.0')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":78,"value":"Buffer = buf.Buffer\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W32","stdlib":"1","titling":"auto"}} 30:"$2f:body"