3e:null 3f:["$","title",null,{"children":"wtf_wikipedia | Vitaly Yastremsky | Observable"}] 45:[] 43:{"id":"71688f321860d146","slug":null,"title":"wtf_wikipedia","thumbnail":"435636f878a2ee6c09e7eaa6fff96946e1ed4e8df85866d2cfed5c1cdf858662","access_level":"public","create_time":"$D2024-02-07T00:05:25.246Z","update_time":"$D2024-02-07T00:05:25.246Z","view_time":null,"first_public_time":"$D2024-02-07T00:05:25.246Z","has_unpublished_edits":null,"fork":{"id":"ee7bf5ce41bc37ce","slug":"wtf_wikipedia","title":"wtf_wikipedia","owner":{"id":"178b998367cf1e11","login":"spencermountain","name":"spencer kelly"}},"stars":0,"starred":false,"owner":{"type":"team","id":"039a72a32a83b05b","login":"zvezdochetag","name":"Vitaly Yastremsky","avatar_url":"https://avatars-next.observableusercontent.com/527e3db558ce86f9dea18847c31fff2b8c3ba42a41778db66342b4299fdda0e9","tier":"starter_2024","bio":"","following":false},"version":647,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"b0b9a3752841dcc9","login":"zvezdochetag","name":"Vitaly Yastremsky","avatar_url":"https://avatars-next.observableusercontent.com/527e3db558ce86f9dea18847c31fff2b8c3ba42a41778db66342b4299fdda0e9","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"b0b9a3752841dcc9","login":"zvezdochetag","name":"Vitaly Yastremsky","avatar_url":"https://avatars-next.observableusercontent.com/527e3db558ce86f9dea18847c31fff2b8c3ba42a41778db66342b4299fdda0e9","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"wtf_wikipedia","cells":[{"id":526,"value":"md`## wtf_wikipedia`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"html`wtf_wikipedia is a javascript library to pull data out of wikipedia articles.\n

laughably difficult thing

\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":287,"value":"html`

\n go ahead:\n

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":46,"value":"viewof page = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":529,"value":"viewof lang = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":192,"value":"doc = await wtf.fetch(page, lang)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":53,"value":"html`\n this method grabs the page's raw wikitext from the wikipedia api and fully parses it.
\n
\n When your promise returns, you have the document in a structured form, that you can play with it, all you want:\n\n\n

\n Template-parsing tutorial\n

\n\n\n

grab the first sentence:

\ngetting the first sentence, of the first section, is as easy as .sentences(0).\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":55,"value":"doc.sentences()[0].text()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":364,"value":"html`or grab its links:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":358,"value":"doc.sentence(0).links().map(l => l.page())","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":366,"value":"html`or flip it into html`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":368,"value":"html`${doc.sentence().html()}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":400,"value":"html`
Sections:
\nsame with a page's sections:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":403,"value":"doc.section(1).title()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":385,"value":"html`page sections are parsed with their title and indentation, so producing a table-of-contents is simple to do:\n
\n doc.sections().map(s=> [s.title(), s.links()])\n
\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":393,"value":"{ \n //list of section titles\n let list = doc.sections().map((s)=>{\n let indent = (' ').repeat(s.indentation())\n let title = s.title() || '(intro)'\n return `
${indent} - ${title}
`\n })\n //how many links each section has\n let counts = doc.sections().map((s)=> {\n let count = s.links().length\n count = count === 0 ? '-' : count+' links'\n return `
${count}
`\n })\n return html`
\n
${list}
\n
${counts}
\n
`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":558,"value":"html`
Paragraphs:
\nParagraphs are blocks of text, along with any images or lists - doc.paragraphs()`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":560,"value":"html`${doc.paragraph(0).text()}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":564,"value":"printList(doc.paragraph(0).links().map(l=>l.page()))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":57,"value":"html`
Categories:
\nhey, no problem - doc.categories()`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":58,"value":"printList(doc.categories())","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":428,"value":"html`
Templates:
\nwikipedia has 634 thousand of custom templates. \n
\nSome of them are invisible metadata, some require inline-replacement of text, others are best to ignore.\n
Fortunately, they tend to follow some patterns.
\n
We parse as many as we can, and ignore the other junk.
\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":430,"value":"doc.templates()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":433,"value":"doc.templates('citation').length","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":442,"value":"doc.templates()[2]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":234,"value":"html`
Infobox data:
\ninfoboxes are particularly rich page templates with semi-structured key-value metadata.
\nEach value of a infobox is parsed as a Sentence object, with all those methods, like .plaintext() or .links()\n
\n
install the wtf-plugin-html plugin
\n
\n
\n npm i wtf-plugin-html\n
\n \n
wtf.plugin(require('wtf-plugin-html'))
\n \n
\n
\n
\n doc.infobox(0).html()\n
\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":140,"value":"html`${doc.infoboxes().map(t=>t.html())}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":483,"value":"html`
Links:
\nexternal and internal links are parsed on each sentence, and can be called from a Section, Document or Infobox.\n\n
here we grab the links of the first section:
\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":476,"value":"printList(doc.section(0).links().map(p => p.page()))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":489,"value":"html`
Custom JSON:
\nwith these concepts, it should be straightforward to go through a wikipedia page and turn it into the data you want.\n
something like this?
\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":491,"value":"{ //generate a list of templates, by each section\n return doc.sections().reduce((h,s)=> {\n h[s.title()||'Intro']= s.templates().map(t=> t.template )\n return h\n },{})\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":61,"value":"md`### -- \t▼ libraries used --\n\n* [wtf-plugin-html](https://github.com/spencermountain/wtf_wikipedia/tree/master/plugins/html)\n\n🎸\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":47,"value":"style = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":108,"value":"printList = function(list){\n if(!list || list.length===0){\n return html`(none)`\n }\n let more=''\n if(list.length>18){\n more=' - and '+(list.length-18)+' more -'\n list=list.slice(0,18)\n }\n list = list.map(l => `
${l}
`)\n return html`
\n ${list.join('')}\n
\n
${more}
\n`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"wtf = {\n let nlp = await require('wtf_wikipedia@latest')\n let plugin = await require('wtf-plugin-html')\n nlp.extend(plugin)\n return nlp\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":630,"value":"wtf.version","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W45","stdlib":"1"}} 44:"$43:body"