gemma3 tokenizer for NodeJS/Browser
npm install @lenml/tokenizer-gemma3@lenml/tokenizers
ts
import { fromPreTrained } from "@lenml/tokenizer-gemma3";
const tokenizer = fromPreTrained();
console.log(
"encode()",
tokenizer.encode("Hello, my dog is cute", null, {
add_special_tokens: true,
})
);
console.log("_encode_text", tokenizer._encode_text("Hello, my dog is cute"));
``