21:null 22:["$","title",null,{"children":"An Analysis of AWS S3 Multipart Upload Performance based on AWS Lambda Execution Memory | Rusty Conover | Observable"}] 27:T4f7,arraystat = arr => { var result = {}; if (arr.length) { // avg result.sum = arr.reduce((a, x) => a + x, 0); result.nb = arr.length; result.avg = result.sum / result.nb; // standard deviation percent = avg(deviation) / avg var sumdeviation = arr.reduce((a, x) => a + Math.abs(x - result.avg), 0); var avgdeviation = sumdeviation / result.nb; result.stddev = avgdeviation / result.avg; // standard deviation percent = avg(deviation) / avg arr = [...arr].sort((a, b) => a - b); result.min = arr[0]; result.q1 = quantile(arr, 0.25); result.median = quantile(arr, 0.5); result.q3 = quantile(arr, 0.75); result.pcustom = quantile(arr, percentile); result.max = arr[arr.length - 1]; result.range = result.max - result.min; // histogram result.histogram = []; var nbBins = 5; var i = result.min; var width = result.range / nbBins; while (nbBins--) { var min = i; var max = i + width; var bin = { min: min, max: max, nb: arr.filter(function(x) { if (nbBins) return x >= min && x < max; else return x >= min; }).length }; i += width; result.histogram.push(bin); } } return result; }28:T98a,picasso.chart({ element: document.querySelector('#container'), data, settings: { scales: { v: { data: { fields: ['min', 'max'] } }, t: { data: { extract: { field: 'Dim' } }, type: 'band' } }, components: [ { type: 'text', text: `1 Gigabyte Uploads - marked at ${percentile} percentile`, layout: { dock: 'top' } }, { type: 'axis', dock: 'left', scale: 't' }, { type: 'axis', dock: 'bottom', scale: 'v' }, { type: 'text', text: 'Multipart Upload Configuration (Lambda Function Memory Size / Size of Part / Parallel Requests)', layout: { dock: 'left' } }, { type: 'text', text: 'Milliseconds', layout: { dock: 'bottom' } }, { type: 'grid-line', x: { scale: 'x' }, ticks: { stroke: 'lightgray', strokeDasharray: '3, 3' } }, { type: 'box', data: { extract: { field: 'Dim', props: { min: { field: 'min' }, start: { field: 'low' }, end: { field: 'high' }, max: { field: 'max' }, med: { field: 'median' } } } }, settings: { major: { scale: 't' }, minor: { scale: 'v' }, orientation: 'horizontal', whisker: { // Optional show: true, // Optional stroke: '#000', // Optional strokeWidth: 1, // Optional width: 1 // Optional }, median: { // Optional show: true, // Optional stroke: '#000', // Optional strokeWidth: 3 // Optional }, line: { // Optional show: true, // Optional stroke: '#000', // Optional strokeWidth: 1 // Optional }, box: { fill: function(d) { console.log(d.datum); return ( chart_colors[ d.datum.value.match(/^(\d)/)[0] % chart_colors.length ] || 'yellow' ); }, width: 0.75 } } } ] } })29:[{"id":"367616d9b63dc42ba1125009b7c5e0f820dac4d78d59db122d0c9cfcb30ae07e6db6d014ca28c10736a7182686e68c9c6227be1a6957aa4d1b50cb1a8d6403d8","href":"https://static.observableusercontent.com/files/367616d9b63dc42ba1125009b7c5e0f820dac4d78d59db122d0c9cfcb30ae07e6db6d014ca28c10736a7182686e68c9c6227be1a6957aa4d1b50cb1a8d6403d8","name":"lambda-runs.log@2.json","type":"application/json","lastModified":1579401654522.021,"size":479700},{"id":"e1c68471080d9d3570a913487fc97d11155dcdf511c7b902c52c1824ca66d74aa290f5f0dd4ccfa51a12d51d1072a5d692ee0b2da75b9703b5aca99dc6544dc2","href":"https://static.observableusercontent.com/files/e1c68471080d9d3570a913487fc97d11155dcdf511c7b902c52c1824ca66d74aa290f5f0dd4ccfa51a12d51d1072a5d692ee0b2da75b9703b5aca99dc6544dc2","name":"lambda-runs.log@6.json","type":"application/json","lastModified":1579410492783.145,"size":640800}] 25:{"id":"b09a57ff75e647c3","slug":"an-analysis-of-aws-s3-multipart-upload-performance-based-on","title":"An Analysis of AWS S3 Multipart Upload Performance based on AWS Lambda Execution Memory","thumbnail":"6bfc60aeff1878960791df7f1b9f0a43fa584801e2d92bd4ad157a30eb7aac09","access_level":"public","create_time":"$D2020-01-19T02:14:50.694Z","update_time":"$D2020-01-19T05:57:49.851Z","view_time":null,"first_public_time":"$D2020-01-19T02:47:01.990Z","has_unpublished_edits":null,"fork":{"id":"eee24cc9bff8cbe4","slug":"an-analysis-of-aws-s3-multipart-upload-parameters-on-upload","title":"An Analysis of AWS S3 Multipart Upload Parameters on Upload Performance","owner":{"id":"7086dbb1b7698f76","login":"rustyconover","name":"Rusty Conover"}},"stars":0,"starred":false,"owner":{"type":"team","id":"7086dbb1b7698f76","login":"rustyconover","name":"Rusty Conover","avatar_url":"https://avatars-next.observableusercontent.com/019e8b78c476a383690d3ab8df8e09fc99d70a5c22a53b885784dcbcc294535c","tier":"starter_2024"},"version":381,"publish_version":381,"role":null,"authored":false,"authors":[{"type":"individual","id":"cf2241b0768bf813","login":"rustyconover","name":"Rusty Conover","avatar_url":"https://avatars-next.observableusercontent.com/019e8b78c476a383690d3ab8df8e09fc99d70a5c22a53b885784dcbcc294535c","tier":null}],"members":[{"type":"individual","id":"cf2241b0768bf813","login":"rustyconover","name":"Rusty Conover","avatar_url":"https://avatars-next.observableusercontent.com/019e8b78c476a383690d3ab8df8e09fc99d70a5c22a53b885784dcbcc294535c","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"titling":"auto","title":"An Analysis of AWS S3 Multipart Upload Performance based on AWS Lambda Execution Memory","cells":[{"id":0,"value":"md`# An Analysis of AWS S3 Multipart Upload Performance based on AWS Lambda Execution Memory\n\nI'm exploring and write performance of S3 Multipart uploads at AWS Lambda by varying:\n1. The upload part size\n2. The number of parallel parts being uploaded at once\n3. The amount of memory that the AWS Lambda Function is executing with.\n\nI've collected results by performing repeated uploads of 1 gigabyte in total data with various parameters.\n\nEach combination has been sampled 60 times.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":313,"value":"md`

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":189,"value":"md`-----\n\n## Graph Parameters:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":281,"value":"viewof percentile = slider({\n min: 0.5,\n max: 1,\n value: 0.90,\n step: 0.01,\n format: \".0%\",\n description:\n \"Percentile mark, formatted as a percentage, 50% is median, 100% is longest value, 90% shows 90th percentile duration.\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":358,"value":"md`Show Lambda Function Memory Size (Mb):`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":360,"value":"viewof memory_size = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":185,"value":"md`Show Part Size (MB):`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":164,"value":"viewof part_size = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":195,"value":"md`Show Parallel Requests:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":192,"value":"viewof parallel_requests = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":202,"value":"md`---------`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":311,"value":"md`\n\nThe box plot shows test configuation with 25% and 75% quartiles shaded for total time to upload the data measured in milliseconds. \n\n## Test Setup Details:\n\n1. Tests are run using an AWS Lambda Function running in us-east-1 to a S3 Bucket also locatedin us-east-1.\n\nThe data set is an attachment to this notebook.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":336,"value":"md`---------------`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":339,"value":"md`## Thoughts:\n\nGoing from 1024 to 2048 Mb of Lambda execution memory shows a reasonable improvement across all upload paramters. 2048 to 3008 Mb does not show much of a change.\n\nStill need to investigate single core CPU performance.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"data = {\n const v = _.groupBy(raw_data, r =>\n [r.memorySize, r.partSize, r.queueSize].join('|')\n );\n return [\n {\n type: \"matrix\",\n data: [[\"Dim\", \"min\", \"low\", \"high\", \"max\", \"median\"]].concat(\n Object.keys(v).map(key_name => {\n const stats = arraystat(v[key_name].map(v => v.duration));\n return [\n [key_name].map(l => {\n const label_parts = l.split(/\\|/);\n return `${label_parts[0]} Mb / ${parseInt(label_parts[1], 10) /\n (1024 * 1024)} Mb / ${label_parts[2]} Reqs`;\n })[0],\n stats.min,\n stats.q1,\n stats.q3,\n stats.max,\n stats.pcustom\n ];\n })\n )\n }\n ];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":50,"value":"$27","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":53,"value":"quantile = (arr, q) => {\n var pos = (arr.length - 1) * q;\n var base = Math.floor(pos);\n var rest = pos - base;\n\n if (typeof arr[base + 1] !== 'undefined') {\n return arr[base] + rest * (arr[base + 1] - arr[base]);\n } else {\n return arr[base];\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":208,"value":"d3 = require(\"d3-array@1\", \"d3-color@1\", \"d3-scale-chromatic@^1.4.0\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":129,"value":"chart_colors = d3[`schemePaired`]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":34,"value":"$28","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"picasso = require(\"picasso.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"raw_data = (await FileAttachment(\"lambda-runs.log@6.json\").text())\n\n .split(\"\\n\")\n .filter(v => v != '')\n .map(v => JSON.parse(v))\n .map(v => {\n delete v.data;\n return v;\n })\n .filter(v => {\n if (part_size === 'all') {\n return true;\n }\n return v.partSize / (1024 * 1024) == parseInt(part_size, 10);\n })\n .filter(v => {\n if (parallel_requests === 'all') {\n return true;\n }\n return v.queueSize == parseInt(parallel_requests, 10);\n })\n .filter(v => {\n if (memory_size === 'all') {\n return true;\n }\n return v.memorySize == parseInt(memory_size, 10);\n })\n\n .sort((a, b) => {\n const m = a.memorySize - b.memorySize;\n if (m != 0) {\n return m;\n }\n const s = a.partSize - b.partSize;\n if (s != 0) {\n return s;\n }\n return a.queueSize - b.queueSize;\n })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"_ = require('lodash')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":305,"value":"import { slider } from \"@jashkenas/inputs\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W29","stdlib":"1"}} 26:"$25:body"