29:null 2a:["$","title",null,{"children":"ex4_0 Running SPARQL queries | Thad Kerosky | Observable"}] 2f:Tc40,button = Inputs.button( [ [ "SPARQL query 1 from notebook with namespaces from g: extract the surnames and email addresses for each person", async (value) => { viewof y.editor.setValue(sparql); viewof y.editor.addPrefixes( Object.fromEntries(await py`dict(${g}.namespaces())`) ); } ], [ "Add namespace prefixes from kg to editor", async (value) => { viewof y.editor.addPrefixes( Object.fromEntries( await py`#depends on ${kg_annotated_nom} dict(${kg}.rdf_graph().namespaces()) ` ) ); // viewof y.editor.setSize("100%", 300); } ], [ "SPARQL query 2 in notebook (kg): recipes that use eggs", (value) => { viewof y.editor.setValue(sparql2); } ], [ "SPARQL query 3 in notebook (kg): flour, eggs, salt", (value) => { viewof y.editor.setValue(sparql3); } ], [ "Advanced wikidata query for species", (value) => { viewof y.editor.setValue(` PREFIX wd: PREFIX wdt: PREFIX wikibase: PREFIX p: PREFIX ps: PREFIX pq: PREFIX rdfs: PREFIX bd: PREFIX schema: SELECT ?item (SAMPLE(?itemLabel) AS ?itemLabel) (SAMPLE(?itemDescription) AS ?itemDescription) (SAMPLE(?pic) AS ?pic) (SAMPLE(?taxonname) AS ?taxonname) (SAMPLE(?taxoncommon) AS ?taxoncommon) (SAMPLE(?instance_of) AS ?instance_of) (SAMPLE(?article) AS ?article) (SAMPLE(?ncbi_on_wikidata) as ?ncbi_on_wikidata) (GROUP_CONCAT(DISTINCT ?country; SEPARATOR=", ") AS ?countries) (GROUP_CONCAT(DISTINCT ?countryLabel; SEPARATOR=", ") AS ?countryLabels) WHERE { # VALUES ?item { # wd:Q734870 #wd:Q4951627 #wd:Q95600208 #wd:Q95623161 # } OPTIONAL { ?item wdt:P18 ?pic. } OPTIONAL { ?item wdt:P225 ?taxonname. } OPTIONAL { ?item wdt:P17 ?country. } OPTIONAL { ?item wdt:P31 ?instance_of. } OPTIONAL { ?item wdt:P17 ?country. } OPTIONAL { ?item wdt:P685 ?ncbi_on_wikidata. } #look for english wikipedia article OPTIONAL { ?article schema:about ?item . ?article schema:inLanguage "en" . ?article schema:isPartOf . } # follow the instancesOf "organisms known by a particular common name" for more taxons OPTIONAL { ?item p:P31 ?statement. ?statement ps:P31 wd:Q55983715. ?statement pq:P642 ?of. ?of wdt:P1843 ?taxoncommon. FILTER((LANG(?taxoncommon)) = "en") ?of wdt:P225 ?taxonname. } OPTIONAL { ?item wdt:P1843 ?taxoncommon. FILTER (lang(?taxoncommon) = 'en') } SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". ?item rdfs:label ?itemLabel. ?item rdfs:description ?itemDescription. ?country rdfs:label ?countryLabel. } } GROUP BY ?item`); } ] ], { label: "Change Editor Contents" } )30:T135c,/* these are dependencies that were built in pyodide docker for kglab in case they could not be installed as wheel packages */ otherKglabDependencies = Promise.all([ await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/CLAPACK.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/asttokens.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/chocolate.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/colorama.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/csvwlib.js" ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/cycler.js" // ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/dataclasses.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/decorator.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/executing.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/fsspec.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/gcsfs.js" ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/gensim.js" // ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/html5lib.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/icecream.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/isodate.js" ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/joblib.js" // ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/keepalive.js" ), // await pyodide.loadPackage("https://thadk.net/pyodide_packages/pyodide_18_1/kglab.js"), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/kiwisolver.js" // ), // await pyodide.loadPackage("https://thadk.net/pyodide_packages/pyodide_18_1/matplotlib.js"), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/micropip.js" // ), // await pyodide.loadPackage("https://thadk.net/pyodide_packages/pyodide_18_1/networkx.js"), // await pyodide.loadPackage("https://thadk.net/pyodide_packages/pyodide_18_1/numpy.js"), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/owlrl.js" ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/packaging.js" // ), // await pyodide.loadPackage("https://thadk.net/pyodide_packages/pyodide_18_1/pandas.js"), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/patsy.js" // ), // await pyodide.loadPackage("https://thadk.net/pyodide_packages/pyodide_18_1/pillow.js"), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/pygments.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/pylev.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/pynvml.js" ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/pyparsing.js" // ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/pyshacl.js" ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/python-dateutil.js" // ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/pytz.js" // ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/rdflib.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/requests.js" ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/scikit-learn.js" // ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/scipy.js" // ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/setuptools.js" // ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/six.js" // ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/smart_open.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/sparqlwrapper.js" ), // await pyodide.loadPackage( // "https://thadk.net/pyodide_packages/pyodide_18_1/statsmodels.js" // ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/tqdm.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/urlpath.js" ), await pyodide.loadPackage( "https://thadk.net/pyodide_packages/pyodide_18_1/webencodings.js" ) ])31:T5c6,// Micropip dependencies are simple "wheel" python packages that do not need to be built in any special way. You just have to use micropip with a PyPi wheel URL or package name. micropipDependenciesPy = py`import micropip, os # Github Pages does not succeed, perhaps mime types, only raw github. micropip.install("https://files.pythonhosted.org/packages/7c/f7/280bf4a3830cf6723535096342785aa3a9a15251b188b6bc592da2dda53f/pslpython-2.2.2-py3-none-any.whl") micropip.install("https://files.pythonhosted.org/packages/af/f4/524415c0744552cce7d8bf3669af78e8a069514405ea4fcbd0cc44733744/urllib3-1.26.7-py2.py3-none-any.whl") micropip.install("https://files.pythonhosted.org/packages/47/84/b06f6729fac8108c5fa3e13cde19b0b3de66ba5538c325496dbe39f5ff8e/charset_normalizer-2.0.9-py3-none-any.whl") micropip.install("https://files.pythonhosted.org/packages/37/45/946c02767aabb873146011e665728b680884cd8fe70dde973c640e45b775/certifi-2021.10.8-py2.py3-none-any.whl") micropip.install("https://files.pythonhosted.org/packages/04/a2/d918dcd22354d8958fe113e1a3630137e0fc8b44859ade3063982eacd2a4/idna-3.3-py3-none-any.whl") micropip.install("language-tags") micropip.install("uritemplate") #micropip.install("https://files.pythonhosted.org/packages/07/d1/e87844ec86e96df7364f21af2263ad6030c0d727660ae89935c7af56a540/pyvis-0.1.9-py3-none-any.whl") micropip.install("https://thadk.net/pyodide_packages/pyodide_18_1/pyvis-0.1.9-py2.py3-none-any.whl") os.listdir("/lib/python3.9/site-packages/") `32:[{"id":"ece1832016566b42c8d77e58bf6acd80bbfd1635e2fb7c60c98e4d16e8b26c6569820229fec0c87e1505451c02262727c7f20032c3bdd12cc610aa3a5a93e0f3","href":"https://static.observableusercontent.com/files/ece1832016566b42c8d77e58bf6acd80bbfd1635e2fb7c60c98e4d16e8b26c6569820229fec0c87e1505451c02262727c7f20032c3bdd12cc610aa3a5a93e0f3","name":"kglab-main.zip","type":"application/zip","lastModified":1640287639517.304,"size":17569732},{"id":"ca346ec38eeef47496a405b01deedae50f447fae2a0c4bed28c285c647d8fa1a5b467476378d3d02260c587cf4e9fa36cf5f6147d5a9f3f1e3aec68f65c45967","href":"https://static.observableusercontent.com/files/ca346ec38eeef47496a405b01deedae50f447fae2a0c4bed28c285c647d8fa1a5b467476378d3d02260c587cf4e9fa36cf5f6147d5a9f3f1e3aec68f65c45967","name":"ex2_0_tmp.ttl","type":"text/turtle","lastModified":1640291424783.287,"size":56780}] 2d:{"id":"0980204d4b2beb1e","slug":"ex4_0-running-sparql-queries","title":"ex4_0 Running SPARQL queries","thumbnail":"21d06e490bbe26c91e6ae1d48a691f66691fe6e7295beaeb9022f9ebbe820181","access_level":"public","create_time":"$D2021-12-23T19:27:19.517Z","update_time":"$D2022-01-03T11:03:57.982Z","view_time":null,"first_public_time":"$D2021-12-29T08:37:51.928Z","has_unpublished_edits":null,"fork":{"id":"654729cd2f27c505","slug":"ex3_0-interactive-graph-visualization-with-pyvis","title":"ex3_0 Interactive graph visualization with pyvis","owner":{"id":"5cd4a42aead2029f","login":"thadk","name":"Thad Kerosky"}},"stars":1,"starred":false,"owner":{"type":"team","id":"5cd4a42aead2029f","login":"thadk","name":"Thad Kerosky","avatar_url":"https://avatars-next.observableusercontent.com/2f2413891eb0ff4669b49685946f849544a1e49e7047923d158638bc30666281","tier":"starter_2024"},"version":1126,"publish_version":1126,"role":null,"authored":false,"authors":[{"type":"individual","id":"be3e6e81af18a319","login":"thadk","name":"Thad Kerosky","avatar_url":"https://avatars-next.observableusercontent.com/2f2413891eb0ff4669b49685946f849544a1e49e7047923d158638bc30666281","tier":null}],"members":[{"type":"individual","id":"be3e6e81af18a319","login":"thadk","name":"Thad Kerosky","avatar_url":"https://avatars-next.observableusercontent.com/2f2413891eb0ff4669b49685946f849544a1e49e7047923d158638bc30666281","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":{"id":"aa93231193d34e3a","slug":"kglab-js-alpha-example","title":"kglab.js (alpha) examples","owner":{"id":"5cd4a42aead2029f","login":"thadk","name":"Thad Kerosky"}},"type":"notebook","body":{"title":"ex4_0 Running SPARQL queries","cells":[{"id":0,"value":"# ex4_0 Running SPARQL queries\n*(ObservableHQ notebook note: `tmp.ttl` from ex2_0 output has been uploaded as attachment `ex2_0_tmp.ttl` )*\n\n[SPARQL](https://www.w3.org/TR/rdf-sparql-query/) is a standard query language for RDF.\nNote: there are also other approaches for queries, such [SHACL](https://www.w3.org/TR/shacl/) the *Shapes Constraint Language*.\n\nIn any case, if you've worked with SQL queries before, SPARQL should seem quite familiar.\n\nOH: \n>\"If you build it, they will SPARQL\"","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":221,"value":"ttl_text = `\n@prefix : .\n@prefix foaf: .\n\n:peep0 a foaf:Person ;\n foaf:givenName \"Alice\" ;\n foaf:familyName \"Nakamoto\" ;\n foaf:phone ;\n foaf:mbox .\n\n:peep1 a foaf:Person ;\n foaf:givenName \"Bob\" ;\n foaf:familyName \"Patel\" ;\n foaf:phone ;\n foaf:mbox .\n\n:peep2 a foaf:Person ;\n foaf:givenName \"Dhanya\" ;\n foaf:familyName \"O'Neill\" ;\n foaf:phone ;\n foaf:mbox .\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":228,"value":"Then construct a graph using `rdflib`:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":237,"value":"g = py`\ng = ${rdflib}.Graph()\ng.parse(data=${ttl_text}, format=\"ttl\") ;\ng`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":240,"value":"py`repr(${g})`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":242,"value":"Now we'll define a SPARQL query to extract the surnames and email addresses for each person in the graph, in descending order by surname:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":246,"value":"sparql = `\n SELECT ?person ?surname ?email\n WHERE {\n ?person foaf:familyName ?surname .\n ?person foaf:mbox ?email .\n }\n ORDER BY DESC(?surname)\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":250,"value":"Then run the SPARQL query and iterate over its result set:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":254,"value":"sparqlquery = py`\ng = ${g}\nout = []\nfor rows in g.query(${sparql}):\n out.append(rows.asdict())\nout\n`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":302,"value":"To access the URLs and literals from those results in more readable form, use the `n3()` function with the graph's namespaces to extract formatted text from each term:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":307,"value":"foaf_demo = py`\nout = []\ng = ${g}\nfor row in g.query(${sparql}):\n person = row[\"person\"].n3(g.namespace_manager)\n surname = eval(row[\"surname\"].n3())\n email = row[\"email\"].n3()\n \n out.append(dict({\"person\": person, \"surname\": surname,\"email\": email}))\nout\n`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":323,"value":"---\n\n## SPARQL queries with `kglab` abstractions","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":328,"value":"Now we'll load our medium size KG from the previous example.\nNote that an additional namespace with the `nom:` prefix gets added at this point.\nWe'll use that in the following examples:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":331,"value":"kg = py`\nnamespaces = {\n \"nom\": \"http://example.org/#\",\n \"wtm\": \"http://purl.org/heals/food/\",\n \"ind\": \"http://purl.org/heals/ingredient/\",\n \"skos\": \"http://www.w3.org/2004/02/skos/core#\",\n }\n\nkg = ${kglab}.KnowledgeGraph(\n name = \"A recipe KG example based on Food.com\",\n base_uri = \"https://www.food.com/recipe/\",\n namespaces = namespaces,\n )\nimport io\n\nwith io.open('ex2_0_tmp.ttl','w',encoding='utf8') as f:\n f.write(${await FileAttachment(\"ex2_0_tmp.ttl\").text()})\nkg.load_rdf(\"ex2_0_tmp.ttl\")\nkg`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":339,"value":"## Identify recipes with special ingredients","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":347,"value":"Let's define a query based on [SPARQL 1.1](https://www.w3.org/TR/sparql11-query/) to look for recipes that use *eggs*:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":352,"value":"sparql2 = `\nSELECT ?recipe ?definition ?time\n WHERE {\n ?recipe rdf:type wtm:Recipe .\n ?recipe skos:definition ?definition .\n ?recipe wtm:hasIngredient ind:ChickenEgg .\n ?recipe wtm:hasCookTime ?time\n }\n ORDER BY DESC(?time)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":355,"value":"Let's take a look at the structure of this query.\nWe can visualize it using `kglab`:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":763,"value":"pyVisFrame(await py`\nsparql = ${sparql2}\npyvis_graph = ${kg}.visualize_query(sparql, notebook=True)\n\npyvis_graph.force_atlas_2based()\npyvis_graph.show(\"tmp.fig06.html\")`)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":360,"value":"Now we'll run the query and iterate through its result set:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":364,"value":"html`${await py`\n${pd}.set_option(\"max_rows\", None)\n\ndf = ${kg}.query_as_df(${sparql2})\ndf.head(20).to_html()\n`}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":376,"value":"That's interesting – notice the outliers based on cooking times, i.e., recipes for *eggnog* or special kinds of *cookies* and *noodles*. Some of these take hours or even days to prepare.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":384,"value":"## Using SPARQL queries and post-processing to create annotations\n\nLet's begin to add [*annotations*](https://www.w3.org/TR/annotation-model/) to the KG.\nIn other words, we'll add descriptions to the recipes to help classify them based on an [*ontology*](https://www.datasciencecentral.com/profiles/blogs/understanding-how-taxonomies-differ-from-ontologies).\n\nWe've created a brief *controlled vocabulary* in RDF called `NOM` in the `dat/nom.ttl` file, which provides part of a simple taxonomy for our recipes.\nTake a look at those definitions – mostly these are distinctions between two kind of flour-based recipes:\n\n * \"pancakes\" -- which use batter in their preparation\n * \"noodles\" -- which use kneading in their preparation\n \nLet's include these definitions in our KG:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":391,"value":"kg_nom = py`\nnamespaces = {\n \"nom\": \"http://example.org/#\",\n \"wtm\": \"http://purl.org/heals/food/\",\n \"ind\": \"http://purl.org/heals/ingredient/\",\n \"skos\": \"http://www.w3.org/2004/02/skos/core#\",\n }\n\nkg_nom = ${kglab}.KnowledgeGraph(\n name = \"A recipe KG example based on Food.com\",\n base_uri = \"https://www.food.com/recipe/\",\n namespaces = namespaces,\n )\nimport io\n\nwith io.open('tmp.ttl','w',encoding='utf8') as f:\n f.write(${await FileAttachment(\"ex2_0_tmp.ttl\").text()})\nkg_nom.load_rdf(\"tmp.ttl\")\n\nwith io.open('nom.ttl','w',encoding='utf8') as f:\n f.write(${await kgzip.file(\"kglab-main/dat/nom.ttl\").text()})\nkg_nom.load_rdf(\"nom.ttl\")\nkg_nom`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":434,"value":"Now we'll query the KG, specifically looking for recipes that use *flour*, *eggs*, *salt* among their ingredients:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":438,"value":"sparql3 = `SELECT ?recipe ?definition\n WHERE {\n ?recipe rdf:type wtm:Recipe .\n ?recipe skos:definition ?definition .\n ?recipe wtm:hasIngredient ind:ChickenEgg .\n ?recipe wtm:hasIngredient ind:AllPurposeFlour .\n ?recipe wtm:hasIngredient ind:Salt\n }\n`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":440,"value":"While looping through a query result set, we can perform some post-processing which would have been quite difficult to express in the SPARQL query language, although relatively simple within a Python loop. For instance, we can compare two points for each recipe in the results:\n\n * Is the word noodle or its close synonyms in the recipe description?\n * Does the recipe use butter?\n \nOur `dat/nom.ttl` vocabulary that we've just added to the KG distinguishes between a `nom:Noodle` and a `nom:Pancake` as two categories of flour-based recipes. Let's run this query to look through the flour-based recipes and determine whether we might be able to craft specific rules to distinguish between noodle-ish foods and pancake-ish foods. For example, using butter is rare among noodle recipes that also use eggs, though more common for pancakes.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":444,"value":"tally_print = py`\ntally = []\nkg = ${kg_nom}\nprints = []\nfor row in kg.query(${sparql3}):\n url = row.recipe\n recipe_name = row.definition\n \n # collect two flags to evaluate our noodle vs. pancake rules\n has_butter = (url, kg.get_ns(\"wtm\").hasIngredient, kg.get_ns(\"ind\").Butter) in kg.rdf_graph()\n sez_noodle = any([x in recipe_name for x in [\"noodle\", \"spaetzle\", \"dumpling\", \"pasta\"]])\n tally.append(( has_butter, sez_noodle ))\n\n if not has_butter and not sez_noodle:\n prints.append(dict({\"url\": url, \"recipe_name\": recipe_name, \"has_butter\": has_butter, \"sez_noodle\": sez_noodle }))\n\n{\"tally\": tally, \"prints\": prints}\n\n`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":573,"value":"Let's tally and visualize a *contingency table* based on what we observed from the query results:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":474,"value":"contingency_table = py`\n# ${packagesPy}\nfrom collections import Counter\n\nplt = ${plt}\n\n#convert the interior 2-element lists coming from javascript back into tuples:\ntally_tuples = [ tuple(x) for x in ${tally_print.get(\"tally\")}]\n\n# warning: \\`mosaic\\` expects data in (x_axis, y_axis), so revert\n# the axis so that this contingency table has the same order\ncontingency_table = dict(Counter(tally_tuples))\n\n# pyodide failed to convert the contingency_table toJs() at this step so could not split out textual and plot into 2 ObservableHQ cells\n\nfrom statsmodels.graphics.mosaicplot import mosaic\n\nfig, ax = plt.subplots(1, 1)\n\nfig, _ = mosaic(\n contingency_table,\n index=[1, 0], \n gap=0.05, \n title=\"recipes\",\n # improvde default cmap\n properties=lambda key: {\"color\": \"yellow\" if key[1] == \"True\" else \"gray\"},\n ax=ax,\n)\n\nax.set_xlabel(\"sez_noddle\")\nax.set_ylabel(\"has_butter\")\nax.set_box_aspect(1)\n(str(contingency_table), plt.show())\n`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":584,"value":"contingency_table[1]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":586,"value":"md`${contingency_table[0]}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":590,"value":"Note the conditional probabilities in this sample:\n\n * `P(sez_noodle|has_butter)` at approximately 1:2\n * `P(has_butter|sez_noodle)` at approximately 1:4\n\nThe latter is significantly lower, so these conditions seem reasonably good to use as filters for identifying which recipes to annotate.\nOf course, not every recipe is either a kind of pancake or noodle, since there are recipes for *omelette*, *Yorkshire pudding*, etc. \n\nWe'll revisit this kind of problem later with more formal, semi-automated methods that leverage machine learning and *human-in-the-loop* approaches.\n\nFor now let's add annotations based on our manual review:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":594,"value":"noodle_ids = [\n \"400\",\n \"86710\",\n \"331765\",\n \"508734\",\n \"320154\",\n \"220361\",\n \"148900\",\n \"317697\",\n \"252783\",\n \"137357\",\n \"1975\",\n \"31041\",\n \"441475\",\n \"261361\",\n \"124106\",\n \"78459\",\n \"358908\",\n \"103964\",\n \"91311\",\n \"497918\",\n \"328388\"\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":596,"value":"pancake_ids = [\n \"277824\",\n \"489\",\n \"4643\",\n \"272746\",\n \"12055\",\n \"124131\",\n \"40772\",\n \"459\",\n \"48178\",\n \"124176\",\n \"61108\",\n \"111008\",\n \"262038\",\n \"458\",\n \"440398\",\n \"157638\"\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":600,"value":"kg_annotated_nom = py`\nkg = ${kg_nom}\nrdflib = ${rdflib}\nfor id in ${noodle_ids}:\n url = f\"https://www.food.com/recipe/{id}\"\n node = rdflib.URIRef(url)\n kg.add(node, kg.get_ns(\"rdf\").type, kg.get_ns(\"nom\").Noodle)\n\nfor id in ${pancake_ids}:\n url = f\"https://www.food.com/recipe/{id}\"\n node = rdflib.URIRef(url)\n kg.add(node, kg.get_ns(\"rdf\").type, kg.get_ns(\"nom\").Pancake)\n\nkg.save_rdf(\"tmp.ttl\")\nkg`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":607,"value":"Now open the `tmp.ttl` file.\nNotice the new annotations for `nom:Noodle` and `nom:Pancake` added to the recipes?","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":684,"value":"{\n // be sure to run the dependency first\n kg_annotated_nom;\n\n return downloadPyoTTLFile(\"tmp.ttl\");\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":664,"value":"*If you expand the next cell, it will be very long. You can save it above instead and look at it in a text editor.*","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":648,"value":"//convert from the UTF-8 arrayBuffer that was saved into something we can see in the browser using the one line function above ab2str\n{\n // be sure to run the dependency first\n kg_annotated_nom;\n const ttlArrayBuffer = await py`\nread = \"\"\nwith open(\"tmp.ttl\", \"rb\") as f:\n read = f.read()\nread\n`;\n return ab2str(ttlArrayBuffer);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":650,"value":"function ab2str(buf) {\n return String.fromCharCode.apply(null, new Uint8Array(buf));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":674,"value":"/* todo: use correct mime type */\nasync function downloadPyoTTLFile(path) {\n // read with javascript, using e.g.\n // pyodide.FS.root.contents[\"tmp.ttl\"].contents\n // but it doesn't allow using a path\n const readArrayBuffer = py`\nread = \"\"\nwith open(${path}, \"rb\") as f:\n read = f.read()\nread\n`;\n const blob = new Blob([await readArrayBuffer], {\n type: \"text/turtle\"\n });\n return DOM.download(blob, path, \"Download the \" + path);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":615,"value":"---\n\n## Exercises","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":617,"value":"**Exercise 1:**\n\nA third category of recipes that use *flour*, *eggs*, *salt* among their ingredients would be \n[*popovers*](https://en.wikipedia.org/wiki/Popover) (US usage) which are much the same as [*Yorkshire pudding*](https://en.wikipedia.org/wiki/Yorkshire_pudding) (UK usage).\n\nExtend the `nom.ttl` vocabulary to include the definition of a `popover`","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":624,"value":"**Exercise 2:**\n\nRevise the queries and post-processing above to identify popover recipes.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":628,"value":"**Exercise 3:**\n\nConstruct a `popover_ids` list for the identified recipes.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":630,"value":"**Exercise 4:**\n \nAnnotate the KG to include `nom:popover` definitions for popovers.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":765,"value":"----\n## Bonus Visual Query Editor","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1011,"value":"queryEditor = view`\n\n \n\n\n

\n ${viewof y}

\n${pyvisSidepanel}\n

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":773,"value":"viewof y = {\n /* set default value */\n // Yasgui.Yasqe.defaults.value = sparql2;\n /* crib from https://observablehq.com/@mootari/codemirror */\n const view = document.createElement(\"div\");\n view.appendChild(\n html`