diff --git a/examples/phi2/phi2-loop.py b/examples/phi2/phi2-streaming.py similarity index 77% rename from examples/phi2/phi2-loop.py rename to examples/phi2/phi2-streaming.py index f874372a0..688fa324a 100644 --- a/examples/phi2/phi2-loop.py +++ b/examples/phi2/phi2-streaming.py @@ -20,24 +20,14 @@ params.input_ids = input_tokens generator=og.Generator(model, params) +tokenizer_stream=tokenizer.create_stream() print("Generator created") print("Output:") - print(prompt, end='', flush=True) - while not generator.is_done(): generator.compute_logits() - - # search.apply_minLength(1) - # search.apply_repetition_penalty(1.0) - generator.generate_next_token_top_p(0.7, 0.6) - -# Print sequence all at once vs as it's decoded: -print(tokenizer.decode(generator.get_sequence(0).get_array())) - -print() -print() + print(tokenizer_stream.decode(generator.get_sequence(0).get_array()[-1]), end='', flush=True)