Use cl100k for LLaMa 3.1

cosmicoptima · Sep 4, 2024 · 788d91b · 788d91b
1 parent c624fd9
commit 788d91b
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 2 deletions.
diff --git a/main.ts b/main.ts
@@ -1479,6 +1479,15 @@ export default class LoomPlugin extends Plugin {
       "gpt-3.5-turbo",
       "gpt-3.5-turbo-0301",
       "gpt-4-base",
+      // TODO: llama 3.1 has "28k additional multilingual tokens", so cl100k is not exactly right
+      "meta-llama/llama-3.1-8b",
+      "meta-llama/llama-3.1-70b",
+      "meta-llama/llama-3.1-405b",
+      "meta-llama/llama-3.1-8b-instruct",
+      "meta-llama/llama-3.1-70b-instruct",
+      "meta-llama/llama-3.1-405b-instruct",
+      "meta-llama/Meta-Llama-3.1-405B",
+      "meta-llama/Meta-Llama-3.1-405B-FP8",
     ];
     const p50kModels = [
       "text-davinci-003",

diff --git a/manifest.json b/manifest.json
@@ -1,7 +1,7 @@
 {
 	"id": "loom",
 	"name": "Loom",
-	"version": "1.22.1",
+	"version": "1.22.2",
 	"minAppVersion": "0.15.0",
 	"description": "Loom in Obsidian",
 	"author": "celeste",

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "obsidian-loom",
-	"version": "1.22.1",
+	"version": "1.22.2",
 	"description": "Loom in Obsidian",
 	"main": "main.js",
 	"scripts": {