21:null 22:["$","title",null,{"children":"An Analysis of AWS S3 Multipart Upload Parameters on Upload Performance | Rusty Conover | Observable"}] 27:T4f7,arraystat = arr => { var result = {}; if (arr.length) { // avg result.sum = arr.reduce((a, x) => a + x, 0); result.nb = arr.length; result.avg = result.sum / result.nb; // standard deviation percent = avg(deviation) / avg var sumdeviation = arr.reduce((a, x) => a + Math.abs(x - result.avg), 0); var avgdeviation = sumdeviation / result.nb; result.stddev = avgdeviation / result.avg; // standard deviation percent = avg(deviation) / avg arr = [...arr].sort((a, b) => a - b); result.min = arr[0]; result.q1 = quantile(arr, 0.25); result.median = quantile(arr, 0.5); result.q3 = quantile(arr, 0.75); result.pcustom = quantile(arr, percentile); result.max = arr[arr.length - 1]; result.range = result.max - result.min; // histogram result.histogram = []; var nbBins = 5; var i = result.min; var width = result.range / nbBins; while (nbBins--) { var min = i; var max = i + width; var bin = { min: min, max: max, nb: arr.filter(function(x) { if (nbBins) return x >= min && x < max; else return x >= min; }).length }; i += width; result.histogram.push(bin); } } return result; }28:T96d,picasso.chart({ element: document.querySelector('#container'), data, settings: { scales: { v: { data: { fields: ['min', 'max'] } }, t: { data: { extract: { field: 'Dim' } }, type: 'band' } }, components: [ { type: 'text', text: `1 Gigabyte Uploads - marked at ${percentile} percentile`, layout: { dock: 'top' } }, { type: 'axis', dock: 'left', scale: 't' }, { type: 'axis', dock: 'bottom', scale: 'v' }, { type: 'text', text: 'Multipart Upload Configuration (Size of Part / Parallel Requests)', layout: { dock: 'left' } }, { type: 'text', text: 'Milliseconds', layout: { dock: 'bottom' } }, { type: 'grid-line', x: { scale: 'x' }, ticks: { stroke: 'lightgray', strokeDasharray: '3, 3' } }, { type: 'box', data: { extract: { field: 'Dim', props: { min: { field: 'min' }, start: { field: 'low' }, end: { field: 'high' }, max: { field: 'max' }, med: { field: 'median' } } } }, settings: { major: { scale: 't' }, minor: { scale: 'v' }, orientation: 'horizontal', whisker: { // Optional show: true, // Optional stroke: '#000', // Optional strokeWidth: 1, // Optional width: 1 // Optional }, median: { // Optional show: true, // Optional stroke: '#000', // Optional strokeWidth: 3 // Optional }, line: { // Optional show: true, // Optional stroke: '#000', // Optional strokeWidth: 1 // Optional }, box: { fill: function(d) { console.log(d.datum); return ( chart_colors[ d.datum.value.match(/^(\d+)/)[0] % chart_colors.length ] || 'yellow' ); }, width: 0.75 } } } ] } })29:[{"id":"0a062382556fe752b74477afa1a1c01925ff4d37154adb688dd2245c8ced6100469cdbbbdd026fa9df523b886eac932eab48160b3dfaf23162b863529aa2c65e","href":"https://static.observableusercontent.com/files/0a062382556fe752b74477afa1a1c01925ff4d37154adb688dd2245c8ced6100469cdbbbdd026fa9df523b886eac932eab48160b3dfaf23162b863529aa2c65e","name":"run.log@12.json","type":"application/json","lastModified":1579182342353.24,"size":76093},{"id":"6df8c6a33ba28a029eabc43d87d34540673ae3c4f60c611acd130c11ccdaef1267247e38bed7ad442e6fe61ac117a99dde93ee2b23a72eeecc3fc0b73cc6ca01","href":"https://static.observableusercontent.com/files/6df8c6a33ba28a029eabc43d87d34540673ae3c4f60c611acd130c11ccdaef1267247e38bed7ad442e6fe61ac117a99dde93ee2b23a72eeecc3fc0b73cc6ca01","name":"all.runs@3.json","type":"application/json","lastModified":1579185007411.324,"size":88810},{"id":"b53b0f0cfcf2752fa708ceb4bbb46d4f639c61c526fa4287a45fa3dbd42774c85945f2528373b0c7276d692095383c543dfff47a8e8305ae4a7045cf5143f312","href":"https://static.observableusercontent.com/files/b53b0f0cfcf2752fa708ceb4bbb46d4f639c61c526fa4287a45fa3dbd42774c85945f2528373b0c7276d692095383c543dfff47a8e8305ae4a7045cf5143f312","name":"all.runs@9.json","type":"application/json","lastModified":1579188129317.674,"size":107468},{"id":"ea7ee9cc3466cf20ecf9d42dc253abf4089decb5a8ee20e080399d9199b0138bd64c27cfe5444e9a72f3741fa3ead8b1c281abe0fed2622ec646549816929070","href":"https://static.observableusercontent.com/files/ea7ee9cc3466cf20ecf9d42dc253abf4089decb5a8ee20e080399d9199b0138bd64c27cfe5444e9a72f3741fa3ead8b1c281abe0fed2622ec646549816929070","name":"all.runs@6.json","type":"application/json","lastModified":1579186376785.909,"size":96840},{"id":"918ef91c50f9bea5dee8764bb91c0ebcefce10b7db18a2ba8a3e669912d8c15570be4892b7dd1f6ed068e5cf2988d4c156b99c3f0e50a2c457140257cf7736a7","href":"https://static.observableusercontent.com/files/918ef91c50f9bea5dee8764bb91c0ebcefce10b7db18a2ba8a3e669912d8c15570be4892b7dd1f6ed068e5cf2988d4c156b99c3f0e50a2c457140257cf7736a7","name":"all.runs@12.json","type":"application/json","lastModified":1579188817496.974,"size":112035},{"id":"cf48a54ef3018b48b1be08165b19087ca56919857f11b39fb3611e1e46b34b139bc03d9160d12c507501bc6b60a5c33d6b8d742f0d93040b4edef449c522ea91","href":"https://static.observableusercontent.com/files/cf48a54ef3018b48b1be08165b19087ca56919857f11b39fb3611e1e46b34b139bc03d9160d12c507501bc6b60a5c33d6b8d742f0d93040b4edef449c522ea91","name":"all.runs@16.json","type":"application/json","lastModified":1579189659583.73,"size":117719},{"id":"a3ae5a3ebf0a6e8a5a3c879afea688371da5884a24fe4e0d4ca75d52cd2163196a10e7120d26d01e48dec7ff0adf1b5b828ada5ee2e54d6d8586d64b858fa39a","href":"https://static.observableusercontent.com/files/a3ae5a3ebf0a6e8a5a3c879afea688371da5884a24fe4e0d4ca75d52cd2163196a10e7120d26d01e48dec7ff0adf1b5b828ada5ee2e54d6d8586d64b858fa39a","name":"all.runs@17.json","type":"application/json","lastModified":1579190591534.92,"size":124245},{"id":"c93beb4da99ee3e3f4c3916974b81a166b4f4c00a8009cc9d2535525c4abd1591e5ea6131e7d7f36ba6687d6ec3fe2bb0f7543c3e44a96f93da7c4d161b16305","href":"https://static.observableusercontent.com/files/c93beb4da99ee3e3f4c3916974b81a166b4f4c00a8009cc9d2535525c4abd1591e5ea6131e7d7f36ba6687d6ec3fe2bb0f7543c3e44a96f93da7c4d161b16305","name":"all.runs@19.json","type":"application/json","lastModified":1579192759049.003,"size":138917},{"id":"5017ee3cee0d56e9432e4c5f9620f0ff1157c7574b11d99c4d0f1cc94823e3bc01e3a67c8d4ab3ab3a1699b4cc09e0dc20ba1f8364dc2cc6b1b88c5cd9b8ed5c","href":"https://static.observableusercontent.com/files/5017ee3cee0d56e9432e4c5f9620f0ff1157c7574b11d99c4d0f1cc94823e3bc01e3a67c8d4ab3ab3a1699b4cc09e0dc20ba1f8364dc2cc6b1b88c5cd9b8ed5c","name":"all.runs@20.json","type":"application/json","lastModified":1579193098060.587,"size":141370},{"id":"13b24e53df5445638d7bb22cae15218d3aa88babd5131393f4700059dd5a8e998ad67fe615ed238dd88bdce3dcceb7900b9db5baf67c8268000594ad683c5b1a","href":"https://static.observableusercontent.com/files/13b24e53df5445638d7bb22cae15218d3aa88babd5131393f4700059dd5a8e998ad67fe615ed238dd88bdce3dcceb7900b9db5baf67c8268000594ad683c5b1a","name":"all.runs@22.json","type":"application/json","lastModified":1579194343758.654,"size":149935}] 25:{"id":"eee24cc9bff8cbe4","slug":"an-analysis-of-aws-s3-multipart-upload-parameters-on-upload","title":"An Analysis of AWS S3 Multipart Upload Parameters on Upload Performance","thumbnail":"8a2cdec2d1fc60601aa9835d0d382fd4cd713e14fbb527a1a202d8c5b1273b01","access_level":"public","create_time":"$D2020-01-16T03:55:54.448Z","update_time":"$D2020-01-16T17:52:59.048Z","view_time":null,"first_public_time":"$D2020-01-16T17:40:02.508Z","has_unpublished_edits":null,"fork":false,"stars":4,"starred":false,"owner":{"type":"team","id":"7086dbb1b7698f76","login":"rustyconover","name":"Rusty Conover","avatar_url":"https://avatars-next.observableusercontent.com/019e8b78c476a383690d3ab8df8e09fc99d70a5c22a53b885784dcbcc294535c","tier":"starter_2024"},"version":344,"publish_version":344,"role":null,"authored":false,"authors":[{"type":"individual","id":"cf2241b0768bf813","login":"rustyconover","name":"Rusty Conover","avatar_url":"https://avatars-next.observableusercontent.com/019e8b78c476a383690d3ab8df8e09fc99d70a5c22a53b885784dcbcc294535c","tier":null}],"members":[{"type":"individual","id":"cf2241b0768bf813","login":"rustyconover","name":"Rusty Conover","avatar_url":"https://avatars-next.observableusercontent.com/019e8b78c476a383690d3ab8df8e09fc99d70a5c22a53b885784dcbcc294535c","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":6,"collection":null,"type":"notebook","body":{"titling":"auto","title":"An Analysis of AWS S3 Multipart Upload Parameters on Upload Performance","cells":[{"id":0,"value":"md`# An Analysis of AWS S3 Multipart Upload Parameters on Upload Performance\n\nI'm interested in exploring and optimizing the performance of S3 Multipart uploads by varying the upload part size and the number of parallel parts being uploaded at once while being concerned with memory usage for buffering.\n\nI've collected results by performing repeated uploads of 1 gigabyte in total data with various parameters.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":313,"value":"md`

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":189,"value":"md`-----\n\n## Graph Parameters:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":281,"value":"viewof percentile = slider({\n min: 0.5,\n max: 1,\n value: 0.90,\n step: 0.01,\n format: \".0%\",\n description:\n \"Percentile mark, formatted as a percentage, 50% is median, 100% is longest value, 90% shows 90th percentile duration.\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":185,"value":"md`Show Part Size (MB):`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":164,"value":"viewof part_size = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":195,"value":"md`Show Parallel Requests:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":192,"value":"viewof parallel_requests = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":202,"value":"md`---------`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":311,"value":"md`\n\nThe box plot shows test configuation with 25% and 75% quartiles shaded for total time to upload the data measured in milliseconds. \n\n## Test Setup Details:\n\n1. Tests are run from an EC2 instance in us-east-1 to a S3 bucket in us-east-1.\n2. EC2 instance type a1.large.\n3. Each parameter set is run 30 times in succession then statistics are calculated.\n\nThe data set is an attachment to this notebook.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":336,"value":"md`---------------`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":339,"value":"md`## Thoughts:\n\nSingle core CPU performance is becoming a bottleneck, I believe this is due to the use of TLS. Will investigate scaling to multiple cores.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"data = {\n const v = _.groupBy(raw_data, r => [r.partSize, r.queueSize].join('|'));\n return [\n {\n type: \"matrix\",\n data: [[\"Dim\", \"min\", \"low\", \"high\", \"max\", \"median\"]].concat(\n Object.keys(v).map(key_name => {\n const stats = arraystat(v[key_name].map(v => v.duration));\n return [\n [key_name].map(l => {\n const label_parts = l.split(/\\|/);\n return `${parseInt(label_parts[0], 10) / (1024 * 1024)} Mb / ${\n label_parts[1]\n } Reqs`;\n })[0],\n stats.min,\n stats.q1,\n stats.q3,\n stats.max,\n stats.pcustom\n ];\n })\n )\n }\n ];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":50,"value":"$27","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":53,"value":"quantile = (arr, q) => {\n var pos = (arr.length - 1) * q;\n var base = Math.floor(pos);\n var rest = pos - base;\n\n if (typeof arr[base + 1] !== 'undefined') {\n return arr[base] + rest * (arr[base + 1] - arr[base]);\n } else {\n return arr[base];\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":208,"value":"d3 = require(\"d3-array@1\", \"d3-color@1\", \"d3-scale-chromatic@^1.4.0\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":129,"value":"chart_colors = d3[`schemePaired`]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":34,"value":"$28","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"picasso = require(\"picasso.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"raw_data = (await FileAttachment(\"all.runs@22.json\").text())\n\n .split(\"\\n\")\n .filter(v => v != '')\n .map(v => JSON.parse(v))\n .map(v => {\n delete v.data;\n return v;\n })\n .filter(v => {\n if (part_size === 'all') {\n return true;\n }\n return v.partSize / (1024 * 1024) == parseInt(part_size, 10);\n })\n .filter(v => {\n if (parallel_requests === 'all') {\n return true;\n }\n return v.queueSize == parseInt(parallel_requests, 10);\n })\n .sort((a, b) => {\n const s = a.partSize - b.partSize;\n if (s != 0) {\n return s;\n }\n return a.queueSize - b.queueSize;\n })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"_ = require('lodash')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":305,"value":"import { slider } from \"@jashkenas/inputs\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W29","stdlib":"1"}} 26:"$25:body"