From 8be1bc519016db517f9fed06d1e4bc600f0658b3 Mon Sep 17 00:00:00 2001
From: Vincent <81693205+chosen-ox@users.noreply.github.com>
Date: Sun, 28 Jan 2024 15:33:58 +0800
Subject: [PATCH] fix: context_length has no effect

---
 ctransformers/llm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ctransformers/llm.py b/ctransformers/llm.py
index 90cdfe2..20395b4 100644
--- a/ctransformers/llm.py
+++ b/ctransformers/llm.py
@@ -562,6 +562,7 @@ def _stream(
             stop = [stop]
 
         tokens = self.tokenize(prompt)
+        max_new_tokens = min(max_new_tokens, self.context_length - len(tokens)
 
         stop_regex = re.compile("|".join(map(re.escape, stop)))
         count = 0