29:null 2a:["$","title",null,{"children":"T5 Train & Test | Stephen Petrides | Observable"}] 2f:[{"id":"715db1223e067f00500780077febc6cebbdd90c151d3d78317c802732252052ab0e367039872ab9c77d6ef99e5f55a0724b35ddc898a1c99cb14c31a379af80a","href":"https://static.observableusercontent.com/files/715db1223e067f00500780077febc6cebbdd90c151d3d78317c802732252052ab0e367039872ab9c77d6ef99e5f55a0724b35ddc898a1c99cb14c31a379af80a","name":"penguins.csv","type":"text/csv","lastModified":1671291402400.389,"size":13506}] 2d:{"id":"a45c5a90335e7ac0","slug":null,"title":"T5 Train & Test","thumbnail":"ecfe4dacd04c004ca4bdf357585d4dd921b9325de542612a59507ee977a6ea9b","access_level":"public","create_time":"$D2022-12-17T15:36:42.400Z","update_time":"$D2022-12-17T22:21:17.304Z","view_time":null,"first_public_time":"$D2022-12-17T15:36:42.400Z","fork":{"id":"e48bf34c52588bc9","slug":"plot-scatterplot","title":"Plot Scatterplot","owner":{"id":"f35c755083683fe5","login":"observablehq","name":"Observable"}},"stars":0,"starred":false,"owner":{"type":"team","id":"3b4417037bdaf52d","login":"sdpetrides","name":"Stephen Petrides","avatar_url":"https://avatars-next.observableusercontent.com/68379d9e1a57d4d8616cc33181b6998689f3a4970008985334333318a8eac803","tier":"starter_2024"},"version":280,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"ef75e9c1a233cd0b","login":"sdpetrides","name":"Stephen Petrides","avatar_url":"https://avatars-next.observableusercontent.com/68379d9e1a57d4d8616cc33181b6998689f3a4970008985334333318a8eac803","tier":null}],"members":[{"type":"individual","id":"ef75e9c1a233cd0b","login":"sdpetrides","name":"Stephen Petrides","avatar_url":"https://avatars-next.observableusercontent.com/68379d9e1a57d4d8616cc33181b6998689f3a4970008985334333318a8eac803","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"T5 Train & Test","cells":[{"id":0,"value":"# T5 Train & Test","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":4,"value":"data = [\n { order: 1, model: \"small\", params: 76961152, training_time: 140.94 },\n { order: 2, model: \"base\", params: 247577856, training_time: 490.93 },\n { order: 3, model: \"large\", params: 783150080, training_time: 1709.90 * 3.5 },\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":205,"value":"

Pre-training T5 Models

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"},{"id":6,"value":"Plot.plot({\n inset: 8,\n grid: true,\n color: {\n type: \"ordinal\",\n scheme: \"category10\",\n legend: true\n },\n x: {\n label: \"Num Parameters\",\n type: \"log\",\n domain: [1e7, 1e9]\n },\n y: {\n label: \"Training Time\",\n type: \"log\",\n domain: [1e2, 1e4]\n },\n marks: [\n Plot.dot(data, {\n x: \"params\",\n y: \"training_time\",\n stroke: \"model\",\n fill: \"model\",\n r: 10,\n sort: {color: \"x\"},\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":124,"value":"cnn_dailymail_data = [\n {\n model: \"small\",\n time: 396,\n rouge_one: 8.13,\n rouge_two: 1.53,\n rouge_l: 7.47\n },\n {\n model: \"base\",\n time: 1147,\n rouge_one: 9.78,\n rouge_two: 1.81,\n rouge_l: 8.7\n },\n {\n model: \"large\",\n time: 2921,\n rouge_one: 13.64,\n rouge_two: 3.45,\n rouge_l: 12.2\n },\n {\n model: \"xl\",\n time: 5500,\n rouge_one: 17.71,\n rouge_two: 4.55,\n rouge_l: 15.12\n }\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":198,"value":"

Evaluating T5 Models on the CNN-Daily Mail Task

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"},{"id":231,"value":"Plot.legend({\n symbol: {\n domain: [\n \"Rouge 1\",\n \"Rouge 2\",\n \"Rouge L\",\n ],\n range: [\"circle\", \"square\", \"triangle\"],\n },\n swatchSize: 16,\n style: {\n fontSize: \"12px\"\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":127,"value":"Plot.plot({\n inset: 8,\n grid: true,\n color: {\n type: \"ordinal\",\n scheme: \"category10\",\n legend: true,\n },\n x: {\n label: \"Training Time (secs)\",\n type: \"log\",\n domain: [1e2, 1e4]\n },\n y: {\n label: \"Performance\",\n domain: [0, 20]\n },\n marks: [\n Plot.dot(cnn_dailymail_data, {\n x: \"time\",\n y: \"rouge_one\",\n fill: \"model\",\n symbol: \"circle\",\n r: 6,\n sort: {color: \"x\"},\n }),\n Plot.dot(cnn_dailymail_data, {\n x: \"time\",\n y: \"rouge_two\",\n fill: \"model\",\n symbol: \"square\",\n r: 6,\n sort: {color: \"x\"},\n }),\n Plot.dot(cnn_dailymail_data, {\n x: \"time\",\n y: \"rouge_l\",\n fill: \"model\",\n symbol: \"triangle\",\n r: 6,\n sort: {color: \"x\"},\n }),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":210,"value":"squad_data = [\n {\n model: \"small\",\n cpu_time: 591,\n gpu_time: 47,\n fm: 1.697\n },\n {\n model: \"base\",\n cpu_time: 1548,\n gpu_time: 75,\n fm: 3.408\n },\n {\n model: \"large\",\n cpu_time: 4182,\n gpu_time: 134,\n fm: 6.456\n },\n {\n model: \"xl\",\n cpu_time: 7950,\n gpu_time: 249,\n fm: 5.442\n }\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":257,"value":"

Evaluating T5 Models on the SQuAD Task

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"},{"id":222,"value":"Plot.plot({\n inset: 8,\n grid: true,\n color: {\n type: \"ordinal\",\n scheme: \"category10\",\n legend: true,\n },\n x: {\n label: \"CPU Time\",\n type: \"log\",\n domain: [1e2, 1e4]\n },\n y: {\n label: \"F1 Score\",\n domain: [0, 8],\n },\n marks: [\n Plot.dot(squad_data, {\n x: \"cpu_time\",\n y: \"fm\",\n fill: \"model\",\n symbol: \"circle\",\n r: 6,\n sort: {color: \"x\"},\n }),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":153,"value":"cnn_dailymail_ft_data = [\n {\n steps: 0,\n rouge_one: 13.64,\n rouge_two: 3.45,\n rouge_l: 12.2\n },\n {\n steps: 1000,\n rouge_one: 26.85,\n rouge_two: 7.79,\n rouge_l: 22.48\n },\n {\n steps: 2000,\n rouge_one: 26.47,\n rouge_two: 7.10,\n rouge_l: 21.78\n },\n {\n steps: 3000,\n rouge_one: 25.81,\n rouge_two: 6.67,\n rouge_l: 21.62\n },\n {\n steps: 4000,\n rouge_one: 29.25,\n rouge_two: 8.86,\n rouge_l: 24.88\n },\n {\n steps: 5000,\n rouge_one: 28.90,\n rouge_two: 8.51,\n rouge_l: 24.34,\n }\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":177,"value":"

Finetuning the T5 Large Model on CNN-Daily Mail Task

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"},{"id":172,"value":"Plot.legend({\n color: {\n domain: [\n \"Rouge 1\",\n \"Rouge 2\",\n \"Rouge L\",\n ],\n scheme: \"category10\",\n },\n legend: \"swatches\",\n className: \"alphabet\",\n swatchSize: 16,\n style: {\n fontSize: \"12px\"\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":156,"value":"Plot.plot({\n inset: 8,\n grid: true,\n scheme: \"category10\",\n x: {\n label: \"Finetuning Steps\",\n },\n y: {\n domain: [0, 30],\n label: \"Performance\",\n },\n marks: [\n Plot.dot(cnn_dailymail_ft_data, {\n x: \"steps\",\n y: \"rouge_one\",\n fill: \"#1f77b4\",\n symbol: \"circle\",\n r: 6,\n }),\n Plot.dot(cnn_dailymail_ft_data, {\n x: \"steps\",\n y: \"rouge_two\",\n fill: \"#ff7f0e\",\n symbol: \"square\",\n r: 6,\n }),\n Plot.dot(cnn_dailymail_ft_data, {\n x: \"steps\",\n y: \"rouge_l\",\n fill: \"#2ca02c\",\n symbol: \"triangle\",\n r: 6,\n }),\n Plot.line(cnn_dailymail_ft_data, {x: \"steps\", y: \"rouge_one\", stroke: \"#1f77b4\"}),\n Plot.line(cnn_dailymail_ft_data, {x: \"steps\", y: \"rouge_two\", stroke: \"#ff7f0e\"}),\n Plot.line(cnn_dailymail_ft_data, {x: \"steps\", y: \"rouge_l\", stroke: \"#2ca02c\"}),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":264,"value":"squad_ft_data = [\n {steps: 0, em: 0.000, fm: 6.456},\n {steps: 1000, em: 23.898, fm: 47.581},\n {steps: 2000, em: 25.052, fm: 48.738},\n {steps: 3000, em: 24.465, fm: 48.904},\n {steps: 4000, em: 24.144, fm: 50.493},\n {steps: 5000, em: 25.676, fm: 50.563},\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":278,"value":"

Finetuning the T5 Large Model on the SQuAD Task

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"},{"id":271,"value":"Plot.legend({\n color: {\n domain: [\n \"Exact Match\",\n \"F1 Score\",\n ],\n scheme: \"category10\",\n },\n legend: \"swatches\",\n className: \"alphabet\",\n swatchSize: 16,\n style: {\n fontSize: \"12px\"\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":267,"value":"Plot.plot({\n inset: 8,\n grid: true,\n scheme: \"category10\",\n x: {\n label: \"Finetuning Steps\",\n },\n y: {\n domain: [0, 55],\n label: \"Performance\",\n },\n marks: [\n Plot.dot(squad_ft_data, {\n x: \"steps\",\n y: \"em\",\n fill: \"#1f77b4\",\n symbol: \"circle\",\n r: 6,\n }),\n Plot.dot(squad_ft_data, {\n x: \"steps\",\n y: \"fm\",\n fill: \"#ff7f0e\",\n symbol: \"square\",\n r: 6,\n }),\n Plot.line(squad_ft_data, {x: \"steps\", y: \"em\", stroke: \"#1f77b4\"}),\n Plot.line(squad_ft_data, {x: \"steps\", y: \"fm\", stroke: \"#ff7f0e\"}),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W2f","stdlib":"1","titling":"auto"}} 2e:"$2d:body"