28:null 29:["$","title",null,{"children":"@statewalker/tknz | Mikhail Kotelnikov | Observable"}] 2e:T416,viewof text = Inputs.textarea({ value: `# Document ## First Section Some examples: - Good tags order: Section! - Wrong tags embedding: Section! - Standalone closing tags are interpreted as special symbols ("<" and ">") - Tags with attributes: text - Tags attributes with non-closed code block: text - Tags containing its own markdown sub-sections: Before...

# Test Section Test Content

... and after ## Second Section \${this is a code}

# Inner section in a div

# Markdown Header in div > div Paragraph in a div. \${ code block... \`\${inner code}\` ...previous code }

# Second Inner Section Text

## Second Section # Another Top-Level Section \${ code block } Final paragraph... `, // label: "Markdown", rows: 15 })2f:T45f,30:[] 2c:{"id":"7e7af1dfd038c551","slug":"statewalker-tknz","title":"@statewalker/tknz","thumbnail":"a7704304a3e5c6ae4ae39978518950e4b0db23ddb25c22e82feeaa2bb8a3da24","access_level":"public","create_time":"$D2024-03-03T22:23:58.741Z","update_time":"$D2026-09-04T06:15:17.600Z","view_time":null,"first_public_time":"$D2024-03-07T07:49:18.983Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"d4c65cdf77f8f4fd","login":"kotelnikov","name":"Mikhail Kotelnikov","avatar_url":"https://avatars-next.observableusercontent.com/529fa53c5b4acc1414273473d4e049e502c44c1bdbb1a0af81f7faa463dcff4f","tier":"starter_2024"},"version":397,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"cdbfd308e8310490","login":"kotelnikov","name":"Mikhail Kotelnikov","avatar_url":"https://avatars-next.observableusercontent.com/529fa53c5b4acc1414273473d4e049e502c44c1bdbb1a0af81f7faa463dcff4f","tier":null}],"members":[{"type":"individual","id":"cdbfd308e8310490","login":"kotelnikov","name":"Mikhail Kotelnikov","avatar_url":"https://avatars-next.observableusercontent.com/529fa53c5b4acc1414273473d4e049e502c44c1bdbb1a0af81f7faa463dcff4f","tier":null,"role":"editor"},{"type":"team","id":"d4c65cdf77f8f4fd","login":"kotelnikov","name":"Mikhail Kotelnikov","avatar_url":"https://avatars-next.observableusercontent.com/529fa53c5b4acc1414273473d4e049e502c44c1bdbb1a0af81f7faa463dcff4f","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"@statewalker/tknz","cells":[{"id":0,"value":"# @statewalker/tknz\n\nhttps://github.com/statewalker/statewalker-tknz\n\n- Produces well-formed AST for Markdown / HTML documents\n- Generates tokens with associated detailed information (start/end positions)\n- Automatically produces hierarchies of document sections based on headers\n- Reads and balances tags\n- Allows to embed code blocks (`\\${code}`) everywhere\n- Allows to add new tokenizers everywhere. The framework guaranties that tokens will be well-formed/closed.\n\nNote: This is **DO NOT** generate HTML or something like that. It just produces a clean Abstract Syntax Tree (AST) annotating the given input. All nodes (\"tokens\") in this AST are associated with their exact start/end positions in the input. It is up to the users to transform this AST to something else. \n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":276,"value":"## Document Sections","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":278,"value":"This library builds a well-formed hierarchy of document sections based on headers.\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":280,"value":"format.block(`\n# Header One\nFirst paragraph\n\n## Section One\nSection one content\n\n### Sub-section A\nText text text\n\n### Sub-section B\nText text text\n\n## Section Two\nText text text\n\n`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":309,"value":"## Code Blocks\n\nJS-template-like code block could be used everywhere - in tags, lists, headers etc.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":312,"value":"Simple code block ${format.inline(\"before ${code} after\")}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":314,"value":"The parser recognizes string templates in code blocks and propely manages thems: ${format.inline(\" before ${A-code `${B-code ` ${even deeper code} ` B-code} A-code } after\")}.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":322,"value":"It manages escaping and curly brackets embedding: ${format.inline(\" before ${A {B {C} {{}} D} E} after } after } after\")}.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":376,"value":"Double curly brackets are (optionally) can be used to define code blocks: ${format.inline(\" before {{A {B ${C} ${ `x ${Y} z` } D} E}} after \")}.","pinned":true,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":216,"value":"## Tags","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":244,"value":"Simple tags: ${format.inline('text')}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":247,"value":"Tags with attributes: ${format.inline('

text

')}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":224,"value":"Tags with quoted attributes: ${format.inline(\n 'text'\n)} ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":254,"value":"Tags hierarchy: ${format.inline(' hello wonderful world! ')}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":256,"value":"Automatically closes non-closed tags: ${format.inline('before hello wonderful world! after')}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":261,"value":"The corresponding AST: \n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":258,"value":"readMd('before hello wonderful world! after')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":270,"value":"Wrong tags embedding: ${format.inline('before hello wonderful world! after ')} (The second tag is splitted to special symbols \"<\" and \">\")","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":267,"value":"Tags with code blocks: ${format.inline('text')}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":251,"value":"Mixed example: ${format.inline(\n 'before text after'\n)} ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":203,"value":"format.inline(\n 'before text after'\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":341,"value":"## Fenced Code Blocks\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":346,"value":"format.block(`\nbefore \n\\`\\`\\`ts\nFirst Typescript Block\n\\`\\`\\`\n\n\\`\\`\\`js\nJavascript Block\n\\`\\`\\`\n\n\\`\\`\\`\nSimple Fenced Block\n\\`\\`\\`\nafter\n`)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":285,"value":"## Lists\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":287,"value":"format.block(`\n# Simple lists\n\n- item one\n- item two\n - sub-item A\n - sub-item B\n- item three\n`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":297,"value":"format.block(`\n# Multi-line list items\n\n- item one\n on multiple\n lines\n- item two\n - sub-item A\n text text\n still in sub-item A\n - sub-item B\n text text\n still in sub-item B\n- item three\n`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":294,"value":"format.block(`\n# Lists with embedding sections\n\n- item one\n- item two\n \n ## Sub-section A\n Text text text\n \n ## Sub-section B\n Text text text\n \n- item three\n`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":300,"value":"format.block(`\n# Lists with Code\n\n- item one\n- item two\n - sub-item A \\${\n code\n block A\n }\n - sub-item B \\${\n code\n block B\n }\n- item three\n`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":219,"value":"## Sandbox","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":14,"value":"ast = readMd(text)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"$2e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":28,"value":"formatAst(ast)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":242,"value":"## Code","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":226,"value":"format = {\n const md = (str) =>\n readMd((Array.isArray(str) ? str : str ? [str] : [\" \"]).join(\"\"));\n return {\n inline: (str) => formatInlineAst(md(str)),\n block: (str) => formatAst(md(str))\n };\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":26,"value":"function formatAst(ast) {\n return htl.html`

${_formatAst(ast)}

`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":201,"value":"function formatInlineAst(ast) {\n return htl.html`${_formatAst(\n ast\n )}`;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":198,"value":"function _formatAst(node) {\n const list = [];\n const children = node.children || [];\n let start = 0;\n for (const child of children) {\n const str = node.value.substring(start, child.start - node.start);\n if (str) list.push(str);\n list.push(_formatAst(child));\n start = child.end - node.start;\n }\n const str = node.value.substring(start);\n if (str) list.push(str);\n const elm = htl.html`${list}`;\n if (node.type === \"HtmlTag\") {\n elm.dataset[\"tag\"] = node.children[0].tagName;\n }\n return elm;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":358,"value":"function newMdReader(...contentReaders) {\n const {\n TokenizerContext,\n newCodeReader,\n newCompositeTokenizer,\n newMdReader,\n newBlockReader\n } = tknz;\n const readCode = newCodeReader();\n const readNewCodeBlock = tknz.newTextFencedBlockReader(\n \"Code\",\n \"{{\",\n \"}}\",\n readCode\n );\n const contentTokenizers = [...contentReaders, readNewCodeBlock, readCode];\n const readContent = newCompositeTokenizer(contentTokenizers);\n\n const options = {\n // readSectionTokens: readContent,\n readHeaderTokens: readCode,\n readOpenTagTokens: readCode,\n readInstructionsTokens: readCode,\n readTagContentTokens: readContent,\n readBlockContent: readContent\n };\n const readToken = newBlockReader(\"MdDocument\", newMdReader(options));\n return (str) => {\n const ctx = new TokenizerContext(str || \"\\n\");\n return readToken(ctx);\n };\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"readMd = newMdReader()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":39,"value":"$2f","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"stylesMd","database":"$undefined"},{"id":377,"value":"import * as tknz from\n\"https://cdn.jsdelivr.net/npm/@statewalker/tknz@0.1.5\"","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W30","stdlib":"1","titling":"auto"}} 2d:"$2c:body"