31:T416,viewof text = Inputs.textarea({ value: `# Document ## First Section Some examples: - Good tags order: Section! - Wrong tags embedding: Section! - Standalone closing tags are interpreted as special symbols ("<" and ">") - Tags with attributes: text - Tags attributes with non-closed code block: text - Tags containing its own markdown sub-sections: Before...

# Test Section Test Content

... and after ## Second Section \${this is a code}

# Inner section in a div

# Markdown Header in div > div Paragraph in a div. \${ code block... \`\${inner code}\` ...previous code }

# Second Inner Section Text

## Second Section # Another Top-Level Section \${ code block } Final paragraph... `, // label: "Markdown", rows: 15 })32:T45f,33:[] 2f:{"title":"@statewalker/tknz","cells":[{"id":0,"value":"# @statewalker/tknz\n\nhttps://github.com/statewalker/statewalker-tknz\n\n- Produces well-formed AST for Markdown / HTML documents\n- Generates tokens with associated detailed information (start/end positions)\n- Automatically produces hierarchies of document sections based on headers\n- Reads and balances tags\n- Allows to embed code blocks (`\\${code}`) everywhere\n- Allows to add new tokenizers everywhere. The framework guaranties that tokens will be well-formed/closed.\n\nNote: This is **DO NOT** generate HTML or something like that. It just produces a clean Abstract Syntax Tree (AST) annotating the given input. All nodes (\"tokens\") in this AST are associated with their exact start/end positions in the input. It is up to the users to transform this AST to something else. \n","pinned":false,"hidden":false,"mode":"md"},{"id":276,"value":"## Document Sections","pinned":false,"hidden":false,"mode":"md"},{"id":278,"value":"This library builds a well-formed hierarchy of document sections based on headers.\n","pinned":false,"hidden":false,"mode":"md"},{"id":280,"value":"format.block(`\n# Header One\nFirst paragraph\n\n## Section One\nSection one content\n\n### Sub-section A\nText text text\n\n### Sub-section B\nText text text\n\n## Section Two\nText text text\n\n`)","pinned":false,"hidden":false,"mode":"ojs"},{"id":309,"value":"## Code Blocks\n\nJS-template-like code block could be used everywhere - in tags, lists, headers etc.","pinned":false,"hidden":false,"mode":"md"},{"id":312,"value":"Simple code block ${format.inline(\"before ${code} after\")}","pinned":false,"hidden":false,"mode":"md"},{"id":314,"value":"The parser recognizes string templates in code blocks and propely manages thems: ${format.inline(\" before ${A-code `${B-code ` ${even deeper code} ` B-code} A-code } after\")}.","pinned":false,"hidden":false,"mode":"md"},{"id":322,"value":"It manages escaping and curly brackets embedding: ${format.inline(\" before ${A {B {C} {{}} D} E} after } after } after\")}.","pinned":false,"hidden":false,"mode":"md"},{"id":376,"value":"Double curly brackets are (optionally) can be used to define code blocks: ${format.inline(\" before {{A {B ${C} ${ `x ${Y} z` } D} E}} after \")}.","pinned":true,"hidden":false,"mode":"md"},{"id":216,"value":"## Tags","pinned":false,"hidden":false,"mode":"md"},{"id":244,"value":"Simple tags: ${format.inline('text')}","pinned":false,"hidden":false,"mode":"md"},{"id":247,"value":"Tags with attributes: ${format.inline('

text

')}","pinned":false,"hidden":false,"mode":"md"},{"id":224,"value":"Tags with quoted attributes: ${format.inline(\n 'text'\n)} ","pinned":false,"hidden":false,"mode":"md"},{"id":254,"value":"Tags hierarchy: ${format.inline(' hello wonderful world! ')}","pinned":false,"hidden":false,"mode":"md"},{"id":256,"value":"Automatically closes non-closed tags: ${format.inline('before hello wonderful world! after')}","pinned":false,"hidden":false,"mode":"md"},{"id":261,"value":"The corresponding AST: \n","pinned":false,"hidden":false,"mode":"md"},{"id":258,"value":"readMd('before hello wonderful world! after')","pinned":false,"hidden":false,"mode":"ojs"},{"id":270,"value":"Wrong tags embedding: ${format.inline('before hello wonderful world! after ')} (The second tag is splitted to special symbols \"<\" and \">\")","pinned":false,"hidden":false,"mode":"md"},{"id":267,"value":"Tags with code blocks: ${format.inline('text')}","pinned":false,"hidden":false,"mode":"md"},{"id":251,"value":"Mixed example: ${format.inline(\n 'before text after'\n)} ","pinned":false,"hidden":false,"mode":"md"},{"id":203,"value":"format.inline(\n 'before text after'\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":341,"value":"## Fenced Code Blocks\n","pinned":false,"hidden":false,"mode":"md"},{"id":346,"value":"format.block(`\nbefore \n\\`\\`\\`ts\nFirst Typescript Block\n\\`\\`\\`\n\n\\`\\`\\`js\nJavascript Block\n\\`\\`\\`\n\n\\`\\`\\`\nSimple Fenced Block\n\\`\\`\\`\nafter\n`)","pinned":true,"hidden":false,"mode":"ojs"},{"id":285,"value":"## Lists\n","pinned":false,"hidden":false,"mode":"md"},{"id":287,"value":"format.block(`\n# Simple lists\n\n- item one\n- item two\n - sub-item A\n - sub-item B\n- item three\n`)","pinned":false,"hidden":false,"mode":"ojs"},{"id":297,"value":"format.block(`\n# Multi-line list items\n\n- item one\n on multiple\n lines\n- item two\n - sub-item A\n text text\n still in sub-item A\n - sub-item B\n text text\n still in sub-item B\n- item three\n`)","pinned":false,"hidden":false,"mode":"ojs"},{"id":294,"value":"format.block(`\n# Lists with embedding sections\n\n- item one\n- item two\n \n ## Sub-section A\n Text text text\n \n ## Sub-section B\n Text text text\n \n- item three\n`)","pinned":false,"hidden":false,"mode":"ojs"},{"id":300,"value":"format.block(`\n# Lists with Code\n\n- item one\n- item two\n - sub-item A \\${\n code\n block A\n }\n - sub-item B \\${\n code\n block B\n }\n- item three\n`)","pinned":false,"hidden":false,"mode":"ojs"},{"id":219,"value":"## Sandbox","pinned":false,"hidden":false,"mode":"md"},{"id":14,"value":"ast = readMd(text)","pinned":false,"hidden":false,"mode":"ojs"},{"id":12,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":28,"value":"formatAst(ast)","pinned":false,"hidden":false,"mode":"ojs"},{"id":242,"value":"## Code","pinned":false,"hidden":false,"mode":"md"},{"id":226,"value":"format = {\n const md = (str) =>\n readMd((Array.isArray(str) ? str : str ? [str] : [\" \"]).join(\"\"));\n return {\n inline: (str) => formatInlineAst(md(str)),\n block: (str) => formatAst(md(str))\n };\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":26,"value":"function formatAst(ast) {\n return htl.html`

${_formatAst(ast)}

`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":201,"value":"function formatInlineAst(ast) {\n return htl.html`${_formatAst(\n ast\n )}`;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":198,"value":"function _formatAst(node) {\n const list = [];\n const children = node.children || [];\n let start = 0;\n for (const child of children) {\n const str = node.value.substring(start, child.start - node.start);\n if (str) list.push(str);\n list.push(_formatAst(child));\n start = child.end - node.start;\n }\n const str = node.value.substring(start);\n if (str) list.push(str);\n const elm = htl.html`${list}`;\n if (node.type === \"HtmlTag\") {\n elm.dataset[\"tag\"] = node.children[0].tagName;\n }\n return elm;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":358,"value":"function newMdReader(...contentReaders) {\n const {\n TokenizerContext,\n newCodeReader,\n newCompositeTokenizer,\n newMdReader,\n newBlockReader\n } = tknz;\n const readCode = newCodeReader();\n const readNewCodeBlock = tknz.newTextFencedBlockReader(\n \"Code\",\n \"{{\",\n \"}}\",\n readCode\n );\n const contentTokenizers = [...contentReaders, readNewCodeBlock, readCode];\n const readContent = newCompositeTokenizer(contentTokenizers);\n\n const options = {\n // readSectionTokens: readContent,\n readHeaderTokens: readCode,\n readOpenTagTokens: readCode,\n readInstructionsTokens: readCode,\n readTagContentTokens: readContent,\n readBlockContent: readContent\n };\n const readToken = newBlockReader(\"MdDocument\", newMdReader(options));\n return (str) => {\n const ctx = new TokenizerContext(str || \"\\n\");\n return readToken(ctx);\n };\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":8,"value":"readMd = newMdReader()","pinned":false,"hidden":false,"mode":"ojs"},{"id":39,"value":"$32","pinned":false,"hidden":false,"mode":"html"},{"id":377,"value":"import * as tknz from\n\"https://cdn.jsdelivr.net/npm/@statewalker/tknz@0.1.5\"","pinned":true,"hidden":false,"mode":"ts"}],"files":"$W33","stdlib":"1","titling":"auto"}