forked from latitudegames/GPT-3-Encoder
-
Notifications
You must be signed in to change notification settings - Fork 1
/
demo.js
24 lines (17 loc) · 729 Bytes
/
demo.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
// import {encode, decode, countTokens, tokenStats} from "gpt-3-encoder"
//or
const {encode, decode, countTokens, tokenStats} = require('../index')
const str = 'This is an example sentence to try encoding out on!'
const encoded = encode(str)
console.log('Encoded this string looks like: ', encoded)
console.log('We can look at each token and what it represents')
for (let token of encoded) {
console.log({token, string: decode([token])})
}
//example count tokens usage
if (countTokens(str) > 5) {
console.log("String is over five tokens, inconcevable");
}
console.log("String Token Stats: ", tokenStats("foo foo bar bar baz"));
const decoded = decode(encoded)
console.log('We can decode it back into:\n', decoded)