lucene-tokenizers

Online Demo

Install:

npm install lucene-tokenizers

$3

var lt = require('lucene-tokenizers');

var ts = new lt.StandardTokenizer();
ts.setReader(new lt.StringReader("Hello World!"));
var res = [], token = null;
while ((token = ts.incrementToken()) !== null) {
var pretty_token = {};
for (var prop in token) {
if(token.hasOwnProperty(prop)) {
pretty_token[prop.substring('_$esjava$'.length)] = token[prop];
}
}
res.push(pretty_token);
}

console.log(JSON.stringify(res, null, 2));

$3

[
{
"text": "Hello",
"type": "",
"start": 0,
"positionIncrement": 1
},
{
"text": "World",
"type": "",
"start": 6,
"positionIncrement": 1
}
]

More about ESJava