Public
Edited
May 29, 2023
3 stars
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
current
Insert cell
status.orderby(aq.desc("run")).view()
Insert cell
md`There are a few tricks here.`
Insert cell
counts = status.join(oldtime.filter(d => !op.match(d.url, "church"))).derive({ua: d => op.match(d.url, "\.ua") ? ".ua" : "other"}).derive({run: d => op.parse_date(d.run), up: d => d.status == "200" ? "Up" : "Down"}).groupby("url").orderby("run").groupby(["run", "up", "ua"]).count().orderby(["run", aq.desc("up")])
Insert cell
Insert cell
Insert cell
counts.derive({r: d => op.month(d.run)}).view()
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
geocoded.features.filter(d => d.properties)
Insert cell
Insert cell
mapboxgl = {
let mapboxgl = await require('mapbox-gl@2.7.0')
mapboxgl.accessToken = 'pk.eyJ1IjoiYmVubXNjaG1pZHQiLCJhIjoiY2wwdm90amluMDV0OTNjcGVhbmRqeTUyYiJ9.iXhGPCFlwsirGzMlS_8xdA'
return mapboxgl
}
Insert cell
Insert cell
Insert cell
Insert cell
status.view()
Insert cell
Insert cell
viewof oldtime = status.filter(d => d.run < '2022-04-25T17:53:20.333497').groupby("url").rollup().view()
Insert cell
y_axis = [...inconsistent.groupby("url").filter(d => d.status==200).rollup({mean: d => op.mean(op.parse_date(d.run))}).orderby("mean").values("url")]
Insert cell
viewof inconsistent = {
const ever = status.filter(d => d.status==200).select("url").dedupe()
return ever.join(status).derive({up: d => d.status == 200 ? 1 : 0, date: d => op.parse_date(d.run)}).groupby(["url"]).filter(d => op.sum(d.up) / op.sum(1) < .98 && op.sum(d.up) / op.sum(1) > .02).derive({up: d => d.up === 1 ? true : false, mean: d => op.mean(d.run * d.up)}).view()
}
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
# IP Addresses

Some code for parsing ipv4 addresses. This is surely wheel reinvention. Anyhow, I turn them into hexadecimal numbers so they're easier to sort.
Insert cell
wider
Insert cell
troubled = new Set(currently_troubled.groupby("url").rollup({count: d => op.sum(d.recent_count)}).orderby(aq.desc("count")).filter(d => d.count > 10).array('url'))
Insert cell
geocoded = {
const jitter = function([long, lat]) {
const theta = Math.random() * 6.28
const r = Math.sqrt(1 - Math.random()) * .1
return [long + Math.sin(theta) * r, lat + .65 * Math.cos(theta) * r]
}
const features = []
let i = 0;
for (let {url, ip, status} of wider) {
const point = geocode(ip)
if (!point.lat) {
continue
}
const datum = {
type: "Feature",
geometry: {
type: "Point",
coordinates: jitter([point.long, point.lat])
},
id: i++,
properties: {
ip, url, place: geocode.place, status, match: url.match(search), troubled : troubled.has(url) ? 1 : 0
}
}
features.push(datum)
}
return {
type: "FeatureCollection",
features
}
}
Insert cell
status.groupby(['ip', 'url']).count().groupby(['url']).count().filter(d => d.count >= 1).orderby(aq.desc('count')).size
Insert cell
function geocode(ip, iplist, statuslist) {
if (ip === null || ip === undefined) {return {}}
if (ip === undefined) {return {}}
if (!ip.split) return {}
const hexed = ip_to_hex(ip)
const loc = d3.bisectLeft(ip_lookup, hexed)
const comp = ip_lookup[loc - 1]
if (comp === undefined) {return {}}
const [start, end] = comp.split("-")
if (end < hexed) {
return {}
}
if (start > hexed) {
return {}
}
return ips.slice(loc - 1, loc).objects()[0]
}
Insert cell
ip_lookup = ips.array('ip')
Insert cell
function ip_to_hex(ip) {
return ip.split("\.").map(d => parseFloat(d).toString(16).padStart(2, "0")).join("")
}
Insert cell
Insert cell
Insert cell
status.filter(d => d.error).antijoin(comparison_join.select("url"), "url").groupby("error").count().orderby(aq.desc('count')).view()
Insert cell
ips.view()
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.

Insert cell
wider = status.groupby('url').derive({status: d => d.status ? d.status : d.error}).rollup({ip: op.mode('ip'), status: op.object_agg('run', 'status')}).filter(d => d.ip).objects()
Insert cell
Insert cell
status.orderby("run").view()
Insert cell
viewof ips = aq.from(aq.fromArrow(pq.readParquet(new Uint8Array(await FileAttachment("ips.parquet").arrayBuffer()))).reify().objects()).view()
Insert cell
Insert cell
Insert cell
Insert cell
d3 = require("d3-array")
Insert cell

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
Learn more