rawData = FileAttachment("output-full.csv").csv()
cleanedData = rawData.filter(
({count, longitude, latitude}) =>
!(isNaN(longitude) && isNaN(latitude)) /* filter out records with invalid coordinates */
&& +count > threshold /* filter out the worksites with low H1B petition count (e.g. < 20) can also filter out wrong addresses */
/* also improve graphic performance */
)
// define a function to aggregate NYC data, since Queens/Manhattan/Long Island/... are all located in NYC
aggregateNYC = (dataset) => {
let totalCount = 0
dataset.forEach(({worksite, count, longitude, latitude}) => {
if (longitude === "-73.9866" && latitude === "40.7306") {
totalCount += +count
}
})
// return the aggregated record
return {worksite: "New York, New York", count: totalCount, longitude: "-73.9866", latitude: "40.7306"}
}
// define a function to aggregate SF data, since there are many wrong addresses like "Sanfrancisco", "San Francsico"...
// their coordinates are the same because Mapbox API are smart enough to detect typos
aggregateSF = (dataset) => {
let totalCount = 0
dataset.forEach(({worksite, count, longitude, latitude}) => {
if (longitude === "-122.419906" && latitude === "37.779026") {
totalCount += +count
}
})
// return the aggregated record
return {worksite: "San Francisco, California", count: totalCount, longitude: "-122.419906", latitude: "37.779026"}
}
// delete all NYC and SF records, then append the aggregated NYC record and SF record
aggregatedData = [...cleanedData
.filter(({worksite, count, longitude, latitude}) =>
!(longitude === "-73.9866" && latitude === "40.7306")
&& !(longitude === "-122.419906" && latitude === "37.779026"))
, aggregateNYC(cleanedData), aggregateSF(cleanedData)]
// re-structure the data to fit mapboxD3 function requirements
data = aggregatedData.map(({worksite, count, longitude, latitude}) => {
return {
type: "Feature",
properties: {worksite, count},
geometry: {
type: "Point",
coordinates: [+longitude, +latitude]
}
}
})
// Custom interpolator
function interpolateTransparentBlue(t) {
const startColor = d3.rgb(64, 196, 255, 0.3); // 70% transparency
const endColor = d3.rgb(2, 119, 189, 0.9); // 10% transparency
return d3.interpolateRgb(startColor, endColor)(t);
}
// Custom color scale with 1 color
function constantColor() {
return d3.rgb(64, 196, 255, 0.3);
}
colorScale = d3
.scaleSequential(constantColor)
.domain([0, d3.max(data, (d) => +d.properties.count)]);
rScale = d3
.scaleSqrt()
.domain([0, d3.max(data, (d) => +d.properties.count)])
.range([0, 30])
// adapted from https://observablehq.com/@john-guerra/mapbox-d3
function* mapboxD3({
width = 900,
height = 600,
features = [], // geoJSON features to draw
id = (d, i) => i,
r = null, // If not null, must be a function that returns the radius in pixels given the datum
fill = null, // If not null, must be a function that returns the color given the datum
stroke = "#777",
mapboxOptions = {
style: "mapbox://styles/mapbox/light-v10",
scrollZoom: true
},
container = html`<div style="height:${height}px; width:${width}px">`, // where to draw
d3Background = null,
delay = null,
transitionDuration = 0,
alphaRestart = 0.3, // Alpha to set before restarting if using simulation collide = true
parentInvalidation = invalidation,
strokeWidth = 1,
strokeOpacity = 1,
controls = false // Should display zooming controls
} = {}) {
let selected = null, // clicked element
loaded = false; // has mapbox finished loading
let rScale = null,
colorScale = null,
_r = r,
simulation = null,
hoverObj;
let dots;
if (!r) {
rScale = d3
.scaleSqrt()
.domain([0, d3.max(features, (d) => d?.properties?.value)])
.range([0, 15]);
_r = (d) => rScale(d?.properties?.value);
}
if (!fill) {
colorScale =
colorScale ||
d3
.scaleSequential(d3.interpolateInferno)
.domain([0, d3.max(features, (d) => d?.properties?.value)]);
fill = (d) => colorScale(d?.properties?.value);
}
yield container; // Give the container dimensions.
const map = (container.value = new mapboxgl.Map({
container,
...mapboxOptions
}));
// Setup our svg layer that we can manipulate with d3
const bb = container.getBoundingClientRect();
const svg = d3
.select(container)
.append("svg")
.attr("class", "d3Mapbox")
.style("position", "absolute")
.style("top", 0)
.style("left", 0)
// .attr("width", bb.width)
// .attr("height", bb.height)
.attr("width", width)
.attr("height", height)
.style("background", d3Background)
.style("pointer-events", "none"); // the svg shouldn't capture mouse events, so we can have pan and zoom from mapbox
//Project any point to map's current state
function projectPoint(lon, lat) {
let point = map.project(new mapboxgl.LngLat(lon, lat));
return [point.x, point.y];
}
//Projection function
let transform = d3.geoTransform({ point: projectPoint });
let path = d3.geoPath().projection(transform);
svg.append("style").text(`
circle.highlight { stroke: red; }
`);
const updateDots = (features) => {
const dots = svg
.selectAll(".feature")
.data(features, id)
.join(r ? "circle" : "path")
.attr("class", "feature")
.attr("stroke", stroke)
.attr("stroke-width", strokeWidth)
.attr("stroke-opacity", strokeOpacity)
.on("click", (evt, d) => (selected = d))
.style("pointer-events", "all");
// hello-tippy needs nodes
if (dots.nodes().length) {
hoverObj = hover(dots, invalidation, { raise: false });
}
return dots;
};
dots = updateDots(features);
const getTransition = () => {
let t = dots;
if (transitionDuration || delay) {
t = t.transition().duration(transitionDuration).delay(delay);
}
return t;
};
const update = () => {
hoverObj && hoverObj.reset();
if (r) {
let t = getTransition();
t.attr("r", _r);
t.attr("cx", (d) => {
const [x, y] = projectPoint.apply({ stream: {} }, [d.geometry.coordinates[0], d.geometry.coordinates[1]]);
return x;
})
.attr("cy", (d) => {
const [x, y] = projectPoint.apply({ stream: {} }, [d.geometry.coordinates[0], d.geometry.coordinates[1]]);
return y;
});
} else {
dots.attr("d", path);
}
dots.attr("fill", fill);
dots.attr("stroke", stroke);
};
// Positioning
//position nodes manually;
if (r) {
dots
.attr("r", _r)
.attr("cx", (d) => {
const [x, y] = projectPoint.apply({ stream: {} }, [d.geometry.coordinates[0], d.geometry.coordinates[1]]);
return x;
})
.attr("cy", (d) => {
const [x, y] = projectPoint.apply({ stream: {} }, [d.geometry.coordinates[0], d.geometry.coordinates[1]]);
return y;
});
} else {
dots.attr("d", path);
}
// Parameters to return
function set() {
container.value = {
transform,
path,
map,
dots,
hoverObj,
selected,
loaded,
update,
simulation,
set r(_) {
rScale = null;
_r = _;
},
get r() {
return r;
},
set fill(_) {
colorScale = null;
fill = _;
},
get fill() {
return fill;
},
set rScale(_) {
rScale = _;
},
get rScale() {
return rScale;
},
set colorScale(_) {
colorScale = _;
},
get colorScale() {
return colorScale;
},
set delay(_) {
delay = _;
},
get delay() {
return delay;
},
set features(_) {
features = _;
dots = updateDots(features);
},
set width(_) {
width = _;
svg.attr("width", width).attr("height", height);
},
set height(_) {
height = _;
svg.attr("width", width).attr("height", height);
}
};
container.dispatchEvent(new Event("input", { bubbles: true }));
}
if (!map) return;
// Every time the map changes, update the dots
map.on("viewreset", update);
map.on("move", update);
map.on("moveend", update);
map.on("load", () => {
loaded = true;
set();
});
parentInvalidation.then(() => {
map.off("viewreset", update);
map.off("move", update);
map.off("moveend", update);
map.remove();
});
if (controls) {
// Add zoom and rotation controls to the map.
map.addControl(new mapboxgl.NavigationControl());
}
// disable map rotation using right click + drag
map.dragRotate.disable();
// disable map rotation using touch rotation gesture
map.touchZoomRotate.disableRotation();
update();
set();
return container;
}
function tipcontent(_d) {
const d = _d.properties;
return htl.html`
<div>Worksite: ${d.worksite}</div>
<div>H1B Petition Count: ${d.count}</div>
`;
}
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.