MBA Candidate at Columbia, Software Engineer, Former Managing Web Editor @ The Michigan Daily. Building things I can see.
eg = EditableGraph({ width, height })
{
if (!page) {
return;
}
run(page);
}
run = async (page) => {
let currentPage = page;
const pageExists = await getNextPage(currentPage);
const philNode = eg.data().nodes.find((n) => n.id === "Philosophy");
// const startCoords = philNode
// ? { x: philNode.x, y: philNode.y }
// : { x: width / 2, y: height / 2 };
if (pageExists) {
if (!found.includes(currentPage)) {
eg.addNode({
id: currentPage,
startNode: true,
size: 20,
// ...startCoords
});
} else {
eg.data().nodes.find((n) => n.id === currentPage).startNode = true;
eg.forceUpdate();
return;
}
} else {
mutable termStatus = "INVALID PAGE NAME";
return;
}
while (!found.includes(currentPage)) {
found.push(currentPage);
await Promises.delay(200);
const { x, y } = eg.data().nodes.find((n) => n.id === currentPage);
const nextPage = await getNextPage(currentPage);
if (!found.includes(nextPage)) {
eg.addNode({
id: nextPage,
size: nextPage === "Philosophy" ? 20 : 12,
x,
y
});
}
eg.addLink(currentPage, nextPage);
currentPage = nextPage;
}
}
found = []
mutable termStatus = ""
async function getNextPage(currentPage) {
const text = await fetch(
`https://en.wikipedia.org/w/api.php?format=json&origin=*&action=parse&prop=text&page=${currentPage}`
).then((response) => response.json());
if (!text.parse) {
return false;
}
const parser = new DOMParser();
const docText = parser.parseFromString(text.parse.text["*"], "text/html");
if (docText.querySelectorAll("p")[0].innerHTML == "Redirect to:") {
return getNextPage(
docText.querySelectorAll("a")[0].href.split("/").slice(-1)[0]
);
}
const pList = docText.querySelectorAll(
".mw-parser-output>p:not(.mw-empty-elt)"
);
// console.log(pList)
const modifiedPList = [...pList]
.filter((d) => d.textContent.trim())
.map((d) => {
// remove small elements
[...d.querySelectorAll("small")].map((d) => d.parentNode.removeChild(d));
// remove span.nowrap elements
[...d.querySelectorAll("span.nowrap")].map((d) =>
d.parentNode.removeChild(d)
);
// remove span.IPA elements
[...d.querySelectorAll("span.IPA")].map((d) =>
d.parentNode.removeChild(d)
);
// remove span#coordinates elements
[...d.querySelectorAll("span#coordinates")].map((d) =>
d.parentNode.removeChild(d)
);
// console.log(d.outerHTML)
// remove parentheses
const removedParens = d.outerHTML.replace(/[^_]\(([^)]+)\)/g, "");
return parser
.parseFromString(removedParens, "text/html")
.querySelectorAll("p")[0];
});
// console.log(modifiedPList)
const aList = modifiedPList
.map((d) => [...d.querySelectorAll("a:not(.extiw)")])
.flat();
// console.log(aList)
return [...aList]
.filter(
(d) =>
d.href.split("/").splice(-2)[0] == "wiki" &&
!d.href.split("/").splice(-1)[0].includes(":")
)[0]
.href.split("/")
.slice(-1)[0]
.split("#")[0];
}
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.