Public
Edited
Aug 28, 2024
dodges = ({
"GritLM/GritLM-7B": [-30, 2],
"nomic-ai/nomic-embed-text-v1.5": [-10, -10],
"intfloat/e5-mistral-7b-instruct": [-50, 3],
"mixedbread-ai/mxbai-embed-large-v1": [50, 11],
"BAAI/bge-large-en-v1.5": [0, 20],
"Salesforce/SFR-Embedding-2_R": [-50, 3.5],
"intfloat/multilingual-e5-large-instruct": [60, -6],
"Alibaba-NLP/gte-Qwen2-7B-instruct": [-50, 4],
"jinaai/jina-embeddings-v2-base-en": [0, -8],
"sentence-transformers/all-MiniLM-L6-v2": [0, -8]
})
chart = {
const svg = d3
.create("svg")
.attr("viewBox", [0, 0, width, height])
.style("max-width", "100%")
.style("height", "auto");
// Add upper left triangle
const triangle = svg
.append("path")
.attr(
"d",
`M${margin.left},${margin.top} L${width / 1.8},${margin.top}
L${margin.left},${height / 2} Z`
)
.attr("fill", "#8DC460")
.attr("opacity", 0.25);
// Add border around the plot area
svg
.append("rect")
.attr("x", margin.left)
.attr("y", margin.top)
.attr("width", width - margin.left - margin.right)
.attr("height", height - margin.top - margin.bottom)
.attr("fill", "none")
.attr("stroke", "black")
.attr("stroke-width", 1);
// Add x-axis
svg
.append("g")
.attr("transform", `translate(0,${height - margin.bottom})`)
.call(xAxis);
// Add y-axis
svg.append("g").attr("transform", `translate(${margin.left},0)`).call(yAxis);
const points = svg
.append("g")
.selectAll("g")
.data(sortedData)
.join("g")
.attr(
"transform",
(d) => `translate(${xScale(d.parameters)},${yScale(d.performance)})`
);
points
.append("circle")
.attr("r", 5)
.attr("fill", (d) => colorScale(d.category));
points
.append("text")
.text((d) => d.name.split("/")[1])
.attr("y", (d) => `${dodges[d.name][1]}px`)
.attr("x", (d) => `${dodges[d.name][0]}px`)
.attr("text-anchor", "middle")
.attr("text-align", "center")
.attr("font-family", "sans-serif")
.attr("font-size", "8px")
.attr("fill", "black");
// Legend (bottom right)
const legend = svg
.append("g")
.attr(
"transform",
`translate(${width - margin.right - 150}, ${height - margin.bottom - 60})`
);
legend
.append("rect")
.attr("width", 140)
.attr("height", 54)
.attr("fill", "white")
.attr("opacity", 0.8);
// Add new legend item for highlighted area
legend
.append("rect")
.attr("x", 0)
.attr("y", 28)
.attr("width", 14)
.attr("height", 14)
.attr("fill", "#8DC460")
.attr("opacity", 0.5);
legend
.append("text")
.attr("x", 20)
.attr("y", 36)
.text("Best perf./cost ratio")
.style("font-size", "12px")
.attr("alignment-baseline", "middle");
svg
.append("text")
.attr("x", width / 2)
.attr("y", margin.top / 2)
.attr("text-anchor", "middle")
.attr("font-size", "16px")
.attr("font-weight", "bold")
.text("MTEB Arena Performance (Elo) vs. Parameters");
return svg.node();
}
data = [
{
name: "GritLM/GritLM-7B",
parameters: 7240000000,
performance: 1090,
emb_size: 4096,
speed: 0.0305386,
embs: 1 / 0.0305386
},
{
name: "nomic-ai/nomic-embed-text-v1.5",
parameters: 137000000,
performance: 1048,
emb_size: 768,
speed: 0.000527132,
embs: 1 / 0.000527132
},
{
name: "intfloat/e5-mistral-7b-instruct",
parameters: 7110000000,
performance: 1076,
emb_size: 4096,
speed: 0.0103813,
embs: 1 / 0.0103813
},
{
name: "mixedbread-ai/mxbai-embed-large-v1",
parameters: 335000000,
performance: 1035,
emb_size: 1024,
speed: 0.000724871,
embs: 1 / 0.000724871
},
{
name: "BAAI/bge-large-en-v1.5",
parameters: 335000000,
performance: 1036,
speed: 0.000760877,
embs: 1 / 0.000760877
},
{
name: "Salesforce/SFR-Embedding-2_R",
parameters: 7110000000,
performance: 1042,
emb_size: 4096,
speed: 0.010284,
embs: 1 / 0.010284
},
{
name: "intfloat/multilingual-e5-large-instruct",
parameters: 560000000,
performance: 1038,
emb_size: 1024,
speed: 0.000982968,
embs: 1 / 0.000982968
},
{
name: "Alibaba-NLP/gte-Qwen2-7B-instruct",
parameters: 7610000000,
performance: 1014,
emb_size: 3584,
speed: 0.00834315,
embs: 1 / 0.00834315
},
{
name: "jinaai/jina-embeddings-v2-base-en",
parameters: 137000000,
performance: 760,
emb_size: 768,
speed: 0.00053287,
embs: 1 / 0.00053287
},
{
name: "sentence-transformers/all-MiniLM-L6-v2",
parameters: 22700000,
performance: 738,
emb_size: 384,
speed: 0.000214246,
embs: 1 / 0.000214246
}
]
width = 700
height = width * (9 / 16)
margin = ({ top: 40, right: 40, bottom: 60, left: 60 })
xScale = d3
.scaleLog()
.domain([
d3.min(data, (d) => d.parameters) / 1.4,
d3.max(data, (d) => d.parameters) * 1.4
])
.range([margin.left, width - margin.right])
yScale = d3
.scaleLog()
.domain([
d3.min(data, (d) => d.performance) * 0.95,
d3.max(data, (d) => d.performance) * 1.05
])
.range([height - margin.bottom, margin.top])
xAxis = (g) =>
g
.attr("transform", `translate(0,${height - margin.bottom})`)
.call(
d3.axisBottom(xScale).tickFormat((d) => {
const formatted = d3.format(".0s")(d).replace("G", "B");
return formatted.endsWith("M")
? formatted
: formatted.replace("B", " B");
})
)
.call((g) =>
g
.selectAll(".tick text")
.attr("transform", "rotate(-45)")
.style("text-anchor", "end")
)
.call((g) => g.select(".domain").remove())
.call((g) =>
g
.append("text")
.attr("x", width / 2) // Center horizontally
.attr("y", margin.bottom - 10) // Adjust vertical position
.attr("fill", "currentColor")
.attr("text-anchor", "middle") // Center the text on its position
.text("Parameters")
)
yAxis = (g) =>
g
.attr("transform", `translate(${margin.left},0)`)
.call(d3.axisLeft(yScale).tickFormat(d3.format("d")).ticks(10))
.call((g) => g.select(".domain").remove())
.call((g) =>
g
.append("text")
.attr("x", -height / 2) // Center vertically
.attr("y", -margin.left + 15) // Adjust horizontal position
.attr("fill", "currentColor")
.attr("text-anchor", "middle") // Center the text on its position
.attr("transform", "rotate(-90)") // Rotate the text
.text("Performance (Elo)")
)
colorScale = d3
.scaleOrdinal()
.domain(["top", "others"])
.range(["forestgreen", "black"])
sortedData = data
.slice()
.sort((a, b) => b.performance - a.performance)
.map((d, i) => ({
...d,
category: d.name.includes("nomic-embed-text-v1.5") ? "top" : "others"
}))
nomicModel = sortedData.find((d) => d.name.includes("nomic-embed-text-v1.5"))
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
