Streaming markdown tokenizer.
README.md:
# Hello World
Some text
var fs = require("fs")
var mdtok = require("md-tokenizer")
var t = mdtok(function (src, token) {
JSON.stringify([token.type, src])
})
fs.createReadStream("README.md").pipe(t)
Output:
["heading 1","# "]
["text","Hello"]
["whitespace"," "]
["text","World"]
["new line","\n"]
["text","Some"]
["whitespace"," "]
["text","text"]
["new line","\n"]