2f:T9de,rawMicrosoft = `Tag,Category,Meaning,Description NN,Noun,Noun (Common),Common noun - singular or mass NNC,Noun,Compound Noun,Compound nouns treated as single unit NNP,Noun,Proper Noun,Proper noun - names of people/places/organizations NST,Noun,Spatio-Temporal Noun,Nouns indicating location or time NC,Noun,Common Noun,Common noun (alternative notation) NV,Noun,Verbal Noun,Noun derived from verb VM,Verb,Main Verb,Main verb VAUX,Verb,Auxiliary Verb,Auxiliary/helping verb VF,Verb,Finite Verb,Finite verb form VNF,Verb,Non-Finite Verb,Non-finite verb form VINF,Verb,Infinitive,Infinitive verb form VNG,Verb,Gerund,Gerund form of verb PRP,Pronoun,Pronoun,Personal/reflexive/inclusive pronouns PPR,Pronoun,Pronominal,Pronominal forms PRL,Pronoun,Relative Pronoun,Relative pronoun (যেটা/which/that) PRF,Pronoun,Reflexive Pronoun,Reflexive pronouns PRC,Pronoun,Reciprocal Pronoun,Reciprocal pronouns PWH,Pronoun,WH-Pronoun,Question pronouns (who/what) JJ,Adjective,Adjective,Adjective JQ,Adjective,Quantifier,Quantifying adjective DEM,Demonstrative,Demonstrative,Demonstrative (this/that) DAB,Demonstrative,Absolute Demonstrative,Absolute demonstrative (এই/this) DRL,Demonstrative,Relative Demonstrative,Relative demonstrative DWH,Demonstrative,WH-Demonstrative,Question demonstrative (which) RB,Adverb,Adverb,Adverb AMN,Adverb,Adverb of Manner,Adverb describing manner/how ALC,Adverb,Adverb of Location,Adverb describing location/where (অবশেষে/finally) CC,Conjunction,Conjunction,Coordinating and subordinating conjunctions CSB,Conjunction,Subordinating Conjunction,Subordinating conjunction (কিন্তু/but) CCD,Conjunction,Coordinating Conjunction,Coordinating conjunction PSP,Postposition,Postposition,Postposition (similar to preposition) PP,Postposition,Post Position,Post position QC,Quantifier,Cardinal Number,Cardinal numbers QF,Quantifier,Quantifier,General quantifiers QO,Quantifier,Ordinal Number,Ordinal numbers WQ,Question,WH-Qualifier,WH-question words NEG,Particle,Negative,Negative particle RP,Particle,Particle,General particle INTF,Particle,Intensifier,Intensifier particle INJ,Particle,Interjection,Interjection CL,Particle,Clitic,Clitic particles LC,Participle,Conditional Participle,Conditional participle LV,Participle,Verbal Participle,Verbal participle SYM,Symbol,Special Symbol,Special symbols PU,Punctuation,Punctuation,Punctuation marks RDP,Other,Reduplication,Reduplication (not documented) BM,Other,Not Documented,Not documented UT,Other,Not Documented,Not documented`30:T6c7,hyderabad = `Tag,Category,Meaning,Description,Example_from_Parser NN,Noun,Common Noun,Singular or mass noun,ছেলেবেলার (childhood) NNP,Noun,Proper Noun,Proper noun for names/places,কলিকাতায় (Kolkata) NST,Noun,Spatio-Temporal,Spatial or temporal location noun,N/A PRP,Pronoun,Pronoun,Personal/demonstrative pronouns,সেটা (that) DEM,Demonstrative,Demonstrative,Demonstrative determiners,এই (this) VM,Verb,Main Verb,Main/lexical verb,নয় (is not) VAUX,Verb,Auxiliary Verb,Auxiliary/helping verb,গেল (went - auxiliary) JJ,Adjective,Adjective,Descriptive adjective,প্রচণ্ড (intense) RB,Adverb,Adverb,Adverb,অবশেষে (finally) QC,Quantifier,Cardinal Number,Cardinal numbers/quantifiers,তিনটি (three) QO,Quantifier,Ordinal Number,Ordinal numbers,প্রথম (first) QF,Quantifier,Quantifier,General quantifiers,N/A CC,Conjunction,Conjunction,Coordinating/subordinating conjunction,কিন্তু (but) PSP,Postposition,Postposition,Postposition (like preposition),N/A RP,Particle,Particle,Particles,N/A NEG,Particle,Negation,Negative particle,N/A INTF,Particle,Intensifier,Intensifier,N/A INJ,Particle,Interjection,Interjection,N/A SYM,Symbol,Symbol/Punctuation,Punctuation and special symbols,. (period) XC,Other,Unknown/Compound,Abbreviations/foreign words/compounds,বি (B. from B.A.) RDP,Other,Reduplication,Reduplicated words,N/A ECH,Other,Echo,Echo words,N/A VGF,Chunk,Verb Group Finite,Finite verb group phrase,N/A VGNF,Chunk,Verb Group Non-Finite,Non-finite verb group phrase,N/A NP,Chunk,Noun Phrase,Noun phrase chunk,N/A RBP,Chunk,Adverb Phrase,Adverb phrase chunk,N/A CCP,Chunk,Conjunction Phrase,Conjunction chunk phrase,N/A`31:Tff6,all_claude = `Tag,Category,Meaning,30-Tag,LDC2010T16,IIIT-26,BNLP,Source NC,Noun,Common Noun,Yes,Yes,No,Yes,"Jahara et al. (2021), LDC2010T16" NN,Noun,Noun (Common),No,Yes,Yes,Yes,"LDC2010T16, IIIT Hyderabad" NNC,Noun,Compound Noun,No,Yes,No,Yes,LDC2010T16 NP,Noun,Proper Noun,Yes,No,No,No,Jahara et al. (2021) NNP,Noun,Proper Noun,No,Yes,Yes,Yes,"LDC2010T16, IIIT Hyderabad" NNPC,Noun,Compound Proper Noun,No,Yes,No,No,LDC2010T16 NV,Noun,Verbal Noun,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" NST,Noun,Spatio-Temporal Noun,Yes,Yes,Yes,No,"All sources" NNT,Noun,Temporal Noun,No,Yes,No,No,LDC2010T16 NLOC,Noun,Noun Location,No,Yes,No,No,LDC2010T16 VM,Verb,Main Verb,Yes,Yes,Yes,Yes,All sources VAUX,Verb,Auxiliary Verb,Yes,Yes,Yes,No,"Jahara et al. (2021), IIIT Hyderabad" VF,Verb,Finite Verb,No,Yes,No,No,LDC2010T16 VNF,Verb,Non-Finite Verb,No,Yes,No,No,LDC2010T16 VINF,Verb,Infinitive,No,Yes,No,No,LDC2010T16 VNG,Verb,Gerund,No,Yes,No,No,LDC2010T16 VBZ,Verb,Verb 3rd Person Singular,No,No,No,Yes,BNLP (custom) PPR,Pronoun,Pronominals,Yes,Yes,No,Yes,"Jahara et al. (2021), LDC2010T16" PRP,Pronoun,Pronoun,No,Yes,Yes,Yes,"LDC2010T16, IIIT Hyderabad" PRF,Pronoun,Reflexive Pronoun,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" PRC,Pronoun,Reciprocal Pronoun,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" PRL,Pronoun,Relative Pronoun,Yes,Yes,No,Yes,"Jahara et al. (2021), LDC2010T16" PWH,Pronoun,WH Pronoun,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" JJ,Adjective,Adjective,Yes,Yes,Yes,Yes,All sources JQ,Adjective,Quantifier,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" JJV,Adjective,Adjective Verbal,No,Yes,No,No,LDC2010T16 DAB,Demonstrative,Absolute Demonstrative,Yes,Yes,No,Yes,"Jahara et al. (2021), LDC2010T16" DEM,Demonstrative,Demonstrative,No,Yes,Yes,No,"LDC2010T16, IIIT Hyderabad" DRL,Demonstrative,Relative Demonstrative,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" DWH,Demonstrative,WH Demonstrative,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" RB,Adverb,Adverb,No,Yes,Yes,No,"LDC2010T16, IIIT Hyderabad" AMN,Adverb,Adverb of Manner,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" ALC,Adverb,Adverb of Location,Yes,Yes,No,Yes,"Jahara et al. (2021), LDC2010T16" CC,Conjunction,Conjunction,No,Yes,Yes,No,"LDC2010T16, IIIT Hyderabad" CCD,Conjunction,Coordinating Conjunction,Yes,Yes,No,Yes,"Jahara et al. (2021), LDC2010T16" CSB,Conjunction,Subordinating Conjunction,Yes,Yes,No,Yes,"Jahara et al. (2021), LDC2010T16" PSP,Postposition,Postposition,No,Yes,No,No,LDC2010T16 PP,Postposition,Post Position,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" QC,Quantifier,Cardinal Number,No,Yes,Yes,No,"LDC2010T16, IIIT Hyderabad" QF,Quantifier,Quantifier,No,Yes,No,No,LDC2010T16 QO,Quantifier,Ordinal Number,No,Yes,Yes,No,"LDC2010T16, IIIT Hyderabad" QJJ,Quantifier,Quantifier Adjective,No,Yes,No,No,LDC2010T16 WQ,Question,WH-Qualifier,No,Yes,No,No,LDC2010T16 NEG,Particle,Negative,No,Yes,No,No,LDC2010T16 CX,Particle,Negation Particle,No,No,No,Yes,BNLP (custom) RP,Particle,Particle,No,Yes,No,No,LDC2010T16 INTF,Particle,Intensifier,No,Yes,No,No,LDC2010T16 INJ,Particle,Interjection,No,Yes,No,No,LDC2010T16 CL,Particle,Clitic,No,Yes,No,No,LDC2010T16 LC,Participle,Conditional Participle,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" LV,Participle,Verbal Participle,Yes,Yes,No,No,"Jahara et al. (2021), LDC2010T16" SYM,Symbol,Special Symbol/Punctuation,No,Yes,Yes,No,"LDC2010T16, IIIT Hyderabad" PU,Punctuation,Punctuation,Yes,No,No,Yes,"Jahara et al. (2021), BNLP" XC,Other,Unknown/Compound/Foreign,No,No,Yes,No,IIIT Hyderabad RDP,Other,Reduplication,No,Yes,No,No,LDC2010T16 BM,Other,Not Documented,No,Yes,No,No,LDC2010T16 UT,Other,Not Documented,No,Yes,No,No,LDC2010T16 NNV,Verb-Noun,Noun from Verb,No,Yes,No,No,LDC2010T16 VGF,Chunk,Verb Group Finite,No,No,Yes,No,IIIT Hyderabad (Chunk) VGNF,Chunk,Verb Group Non-Finite,No,No,Yes,No,IIIT Hyderabad (Chunk) NP_CHUNK,Chunk,Noun Phrase,No,No,Yes,No,IIIT Hyderabad (Chunk) RBP,Chunk,Adverb Phrase,No,No,Yes,No,IIIT Hyderabad (Chunk) CCP,Chunk,Conjunction Chunk Phrase,No,No,Yes,No,IIIT Hyderabad (Chunk)`32:[] 2d:{"title":"Tagsets I","cells":[{"id":0,"value":"# Tagsets I","pinned":false,"hidden":false,"mode":"md"},{"id":21,"value":"## BNLP Tags","pinned":false,"hidden":false,"mode":"md"},{"id":13,"value":"Source [raw30]:\nJahara, F., Barua, A., Iqbal, M. A., Das, A., Sharif, O., Hoque, M. M., & Sarker, I. H. (2021). Towards POS Tagging Methods for Bengali Language: A Comparative Analysis. In Intelligent Computing and Optimization (pp. 1058-1071). Springer. \n\nDOI/Link: https://link.springer.com/chapter/10.1007/978-3-030-68154-8_93 \n\nDataset Details:\n\nTotal corpus size: 7,390 sentences\nTotal tokens: 115K tokens (approximately 115,000 words)\nThis is their annotated Bengali corpus used for evaluating 16 different POS tagging methods","pinned":false,"hidden":false,"mode":"md"},{"id":8,"value":"Inputs.table(d3.csvParse(raw30, d3.autoType))","pinned":false,"hidden":false,"mode":"ojs"},{"id":34,"value":"## More Common Tagset (Microsoft India etc.)","pinned":false,"hidden":false,"mode":"md"},{"id":29,"value":"Source 2: The LDC/Microsoft Research India Tagset\nDataset:\nBali, K., Choudhury, M., & Biswas, P. (2010). Indian Language Part-of-Speech Tagset: Bengali [LDC2010T16]. Linguistic Data Consortium, Philadelphia.\nISBN: 1-58563-561-8\nLink: https://catalog.ldc.upenn.edu/LDC2010T16\nAnnotation Guidelines (PDF): The detailed annotation guidelines are available at https://catalog.ldc.upenn.edu/docs/LDC2010T16/Annotation_Guidelines_for_Bangla.pdf\nThis is the foundational tagset developed by Microsoft Research India as part of the Indian Language Part-of-Speech Tagset (IL-POST) project, which provides the hierarchical framework (Categories, Types, and Attributes) that many Bengali POS taggers use.","pinned":false,"hidden":false,"mode":"md"},{"id":36,"value":"Inputs.table(d3.csvParse(rawMicrosoft, d3.autoType))","pinned":false,"hidden":false,"mode":"ojs"},{"id":78,"value":"## Yet another parser \nhttps://ltrc.iiit.ac.in/analyzer/bengali/\n","pinned":false,"hidden":false,"mode":"md"},{"id":88,"value":"Inputs.table(d3.csvParse(hyderabad, d3.autoType))","pinned":true,"hidden":false,"mode":"ojs"},{"id":63,"value":"## JOINED TABLE, courtesy Claude","pinned":false,"hidden":false,"mode":"md"},{"id":58,"value":"Inputs.table(d3.csvParse(all_claude, d3.autoType))","pinned":false,"hidden":false,"mode":"ojs"},{"id":65,"value":"
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
","pinned":false,"hidden":false,"mode":"html"},{"id":54,"value":"// Full outer join on Category and Meaning\nfullJoin = {\n // Create a composite key from Category and Meaning\n const map1 = new Map(tagset1.map((d) => [`${d.Category}|${d.Meaning}`, d]));\n const map2 = new Map(tagset2.map((d) => [`${d.Category}|${d.Meaning}`, d]));\n\n // Get all unique combinations of Category|Meaning\n const allKeys = new Set([...map1.keys(), ...map2.keys()]);\n\n // Build the joined result\n return Array.from(allKeys)\n .map((key) => {\n const [category, meaning] = key.split(\"|\");\n const row1 = map1.get(key);\n const row2 = map2.get(key);\n\n return {\n Category: category,\n Meaning: meaning,\n Tag1: row1?.Tag ?? null,\n Tag2: row2?.Tag ?? null,\n Token_Count: row1?.Token_Count ?? null,\n Description: row2?.Description ?? null\n };\n })\n .sort(\n (a, b) =>\n a.Category.localeCompare(b.Category) ||\n a.Meaning.localeCompare(b.Meaning)\n );\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":49,"value":"Inputs.table(joinedData)","pinned":false,"hidden":false,"mode":"ojs"},{"id":51,"value":"joinedData = tagset1.map((row1) => {\n const match = tagset2.find((row2) => row2.Meaning === row1.Meaning);\n return match ? { ...row1, ...match } : row1;\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":47,"value":"// Inner join - only matching records\ninnerJoin = {\n const map2 = new Map(tagset2.map((d) => [d.Meaning, d]));\n return tagset1\n .filter((d) => map2.has(d.Meaning))\n .map((d) => ({ ...d, ...map2.get(d.Meaning) }));\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":40,"value":"tagset1 = d3.csvParse(raw30, d3.autoType)","pinned":true,"hidden":false,"mode":"ojs"},{"id":42,"value":"tagset2 = d3.csvParse(rawMicrosoft, d3.autoType)","pinned":true,"hidden":false,"mode":"ojs"},{"id":85,"value":"tagset3 = d3.csvParse(hyderabad, d3.autoType)","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"raw30 = `Tag,Category,Meaning,Token_Count\nNC,Noun,Common Noun,30819\nNP,Noun,Proper Noun,7994\nNV,Noun,Verbal Noun,2985\nNST,Noun,Spatio-Temporal Noun,2627\nVM,Verb,Main Verb,12062\nVAUX,Verb,Auxiliary Verb,2230\nPPR,Pronoun,Pronominals,5137\nPRF,Pronoun,Reflexive Pronoun,362\nPRC,Pronoun,Reciprocal Pronoun,15\nPRL,Pronoun,Relative Pronoun,448\nPWH,Pronoun,WH Pronoun,447\nJJ,Nominal Modifier,Adjective,9377\nJQ,Nominal Modifier,Quantifier,4955\nDAB,Demonstrative,Absolute Demonstrative,2421\nDRL,Demonstrative,Relative Demonstrative,400\nDWH,Demonstrative,WH Demonstrative,55\nAMN,Adverb,Adverb of Manner,1995\nALC,Adverb,Adverb of Location,1970\nLV,Participle,Verbal Participle,72\nLC,Participle,Conditional Participle,501\nPP,Post Position,Post Position,3989\nCCD,Particle,Coordinating Particle,2899\nCSB,Particle,Subordinating Particle,2051\nC_OTHER,Particle,Other Particles,1754\nCX,Particle,Negation/Negative Particle,N/A\nPU,Punctuation,Punctuation,N/A`","pinned":false,"hidden":false,"mode":"ojs"},{"id":31,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs"},{"id":80,"value":"$30","pinned":true,"hidden":false,"mode":"ojs"},{"id":82,"value":"$31","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}