From ccb5e56743fbc7c61c8adf7817d3073e96e07f43 Mon Sep 17 00:00:00 2001 From: Salar Hosseini Date: Tue, 21 May 2024 20:23:12 +0000 Subject: [PATCH] #5383: [Falcon7b] Add perf and output verification settings to demo and add demo to CI Signed-off-by: Salar Hosseini --- models/demos/falcon7b/demo/demo.py | 98 ++++++++++++++++--- .../demos/falcon7b/tests/expected_output.json | 1 - models/demos/grayskull/falcon7b/README.md | 6 +- .../grayskull/falcon7b/demo_grayskull.py | 22 ++++- .../falcon7b/expected_greedy_output.json | 1 + models/demos/t3000/falcon7b/README.md | 6 +- models/demos/t3000/falcon7b/demo_t3000.py | 26 ++++- .../falcon7b/expected_greedy_output.json | 1 + models/demos/wormhole/falcon7b/README.md | 6 +- .../demos/wormhole/falcon7b/demo_wormhole.py | 26 ++++- .../falcon7b/expected_greedy_output.json | 1 + tests/scripts/t3000/run_t3000_demo_tests.sh | 21 +++- 12 files changed, 183 insertions(+), 32 deletions(-) delete mode 100644 models/demos/falcon7b/tests/expected_output.json create mode 100644 models/demos/grayskull/falcon7b/expected_greedy_output.json create mode 100644 models/demos/t3000/falcon7b/expected_greedy_output.json create mode 100644 models/demos/wormhole/falcon7b/expected_greedy_output.json diff --git a/models/demos/falcon7b/demo/demo.py b/models/demos/falcon7b/demo/demo.py index 490b5dd041b..23e82f41509 100644 --- a/models/demos/falcon7b/demo/demo.py +++ b/models/demos/falcon7b/demo/demo.py @@ -148,7 +148,19 @@ def run_falcon_demo_kv( num_layers=32, perf_mode=False, # Option to measure perf using max seq length (with invalid outputs) greedy_sampling=False, # Option to use greedy decoding instead of top-k/p + expected_perf_prefill_decode=None, # Expected perf (t/s) for prefill and decode in perf mode + expected_greedy_output_path=None, # Path for expected outputs for greedy decoding + save_generated_text_path=None, # If provided, save generated text to this path (e.g. set to expected_greedy_output_path to update expected output) ): + assert not (expected_perf_prefill_decode and expected_greedy_output_path), "Cannot verify both perf and output!" + assert not (perf_mode and save_generated_text_path), "Cannot save generated text in perf mode!" + if expected_greedy_output_path is not None: + assert ( + not perf_mode and greedy_sampling + ), "Output verification only supported for greedy sampling in default mode!" + elif expected_perf_prefill_decode is not None: + assert perf_mode, "Performance verification is only supported for perf mode!" + disable_persistent_kernel_cache() disable_compilation_reports() @@ -402,7 +414,8 @@ def run_falcon_demo_kv( logger.info("Finished inference prefill stage!") num_users_generated_prefill = num_users if not perf_mode else (N - N_warmup) * num_devices - generated_ids = torch.concat((prefill_ids[..., :num_input_tokens], output_ids), dim=1) + if not perf_mode: + generated_ids = torch.concat((prefill_ids[..., :num_input_tokens], output_ids), dim=1) ### Inference run decode ### logger.info("Running inference decode stage...") @@ -422,9 +435,12 @@ def run_falcon_demo_kv( N = max_seq_len - num_input_tokens N_warmup = 0 else: - N = 15 - N_warmup = 5 - for output_token_index in range(N): + N = 30 + N_warmup = 10 + print_per_generated_token = ( + expected_greedy_output_path is None + ) # print per generated token if not verifying outputs + for output_token_index in range(N) if print_per_generated_token else tqdm(range(N), desc="Generating tokens"): time_decode_inference_start = time.time() ( tt_decode_input_ids, @@ -473,9 +489,9 @@ def run_falcon_demo_kv( generated_ids = torch.concat((generated_ids, decode_ids[:num_users]), dim=1) kv_cache_len += 1 - # TODO: Remove if we don't want to print per generated token - os.system("clear") - print_output_prompts(generated_ids, tokenizer, batch_size) + if print_per_generated_token: + os.system("clear") + print_output_prompts(generated_ids, tokenizer, batch_size) logger.info("Finished inference decode stage!") num_tokens_generated_decode = global_batch * (output_token_index - N_warmup + 1) @@ -483,12 +499,17 @@ def run_falcon_demo_kv( if not perf_mode: print_output_prompts(generated_ids, tokenizer, batch_size) + generated_text = tokenizer.batch_decode(generated_ids.tolist()) + + if save_generated_text_path is not None: + with open(save_generated_text_path, "w") as f: + json.dump(generated_text, f) + else: + generated_text = None for device in devices: device.disable_and_clear_program_cache() - generated_text = tokenizer.batch_decode(generated_ids.tolist()) - measurements = { "preprocessing": profiler.get("tokenizing_inputs"), "loading_weights": profiler.get("loading_weights"), @@ -500,8 +521,11 @@ def run_falcon_demo_kv( "inference_prefill": time_prefill_inference, "inference_decode": time_decode_inference, "inference_total": time_prefill_inference + time_decode_inference, - "inference_throughput_prefill": num_users_generated_prefill / time_prefill_inference, - "inference_throughput_decode": num_tokens_generated_decode / time_decode_inference, + "inference_user_throughput_prefill": num_users_generated_prefill / time_prefill_inference, # users/s + "inference_token_throughput_prefill": num_users_generated_prefill + / time_prefill_inference + * prefill_ids.shape[1], # tokens/s + "inference_token_throughput_decode": num_tokens_generated_decode / time_decode_inference, # tokens/s } logger.info(f"pre processing: {round(measurements['preprocessing'], 5)} s") @@ -516,13 +540,57 @@ def run_falcon_demo_kv( logger.info(f"prefill inference time: {round(measurements['inference_prefill'], 5)} s") logger.info(f"decode inference time: {round(measurements['inference_decode'], 5)} s") logger.info(f"total inference time: {round(measurements['inference_total'], 5)} s") - logger.info(f"inference throughput prefill: {round(measurements['inference_throughput_prefill'], 5)} users/s") + logger.info(f"inference throughput prefill: {round(measurements['inference_user_throughput_prefill'], 5)} users/s") logger.info( - f"inference throughput prefill | seq_len={prefill_ids.shape[1]} : {round(measurements['inference_throughput_prefill']*prefill_ids.shape[1], 5)} tok/s" + f"inference throughput prefill | seq_len={prefill_ids.shape[1]} : {round(measurements['inference_token_throughput_prefill'], 5)} tok/s" ) - logger.info(f"inference throughput decode: {round(measurements['inference_throughput_decode'], 5)} tok/s") + logger.info(f"inference throughput decode: {round(measurements['inference_token_throughput_decode'], 5)} tok/s") logger.info( - f"inference throughput decode (per user): {round(measurements['inference_throughput_decode']/global_batch, 5)} tok/s/user" + f"inference throughput decode (per user): {round(measurements['inference_token_throughput_decode']/global_batch, 5)} tok/s/user" + ) + + # Verify output or perf if expected values are provided + perf_prefill_decode = [ + measurements["inference_token_throughput_prefill"], + measurements["inference_token_throughput_decode"], + ] + verify_output_or_perf( + generated_text, perf_prefill_decode, expected_greedy_output_path, expected_perf_prefill_decode ) return generated_text, measurements + + +def verify_output_or_perf( + generated_text, perf_prefill_decode, expected_greedy_output_path, expected_perf_prefill_decode +): + assert expected_perf_prefill_decode is None or expected_greedy_output_path is None + if expected_perf_prefill_decode is not None: + does_pass = True + if perf_prefill_decode[0] < expected_perf_prefill_decode[0]: + does_pass = False + logger.warning( + f"Prefill perf {perf_prefill_decode[0]} is lower than expected {expected_perf_prefill_decode[0]}" + ) + if perf_prefill_decode[1] < expected_perf_prefill_decode[1]: + does_pass = False + logger.warning( + f"Decode perf {perf_prefill_decode[1]} is lower than expected {expected_perf_prefill_decode[1]}" + ) + if does_pass: + logger.info("Perf Check Passed!") + else: + logger.warning("Perf Check Failed!") + assert ( + does_pass + ), f"Prefill or decode perf is lower than {expected_perf_prefill_decode}. See earlier warnings for more details." + elif expected_greedy_output_path is not None: + with open(expected_greedy_output_path, "r") as f: + expected_output = json.load(f) + does_pass = generated_text == expected_output + if does_pass: + logger.info("Output Check Passed!") + else: + assert ( + does_pass + ), f"Generated text does not match expected output! \n\n Generated text:\n {generated_text} \n\n Expected output:\n {expected_output}" diff --git a/models/demos/falcon7b/tests/expected_output.json b/models/demos/falcon7b/tests/expected_output.json deleted file mode 100644 index 3c983743734..00000000000 --- a/models/demos/falcon7b/tests/expected_output.json +++ /dev/null @@ -1 +0,0 @@ -{"0": "write a short poem about London in English\nLondon, city of dreams,\nA skyline filled with stories to tell.\nFrom the Tower to the Thames,\nA bustling heart beats in every street.\n\nThe hustle and bustle of the city,\nA vibrant symphony of a busy life.\nThe iconic landmarks, a skyline so grand,\nA melting pot of cultures, London is a must-see land. ", "1": "write a short poem about Madrid in Spanish\nMadrid, ciudad hermos\u00edsima,\nEn su cielo, un sol resplandeciente.\nMi amor por ti, nunca se ha menguado,\nEn mi alma, una nostalgia infinita.\nMadrid, ciudad hermos\u00edsima,\nEn su cielo, un sol resplandeciente.\nMi amor por ti, nunca se ha menguado,\nEn mi alma, una nostalgia infinita. ", "2": "write a short poem about Madrid in English\nMadrid, city of dreams,\nSparkling fountains, iconic scenes,\nA city of culture, art, and history,\nWhere the past meets the present, and the future is wide open. ", "3": "write a short poem about Paris in English\nParis, the City of Lights,\nA beauty so divine,\nIts skyline, a dazzling sight,\nA heart that's ever kind.\n\nThe Eiffel Tower, a masterpiece,\nIts grandeur, a true delight,\nA symbol of love, a timeless tale,\nA feeling of joy, a heavenly sight.\n\nThe Seine, a timeless river,\nIts gilded banks, a shimmering sight,\nA muse, a legend, a timeless tale,\nA feeling of joy, a heavenly sight.\n\nNotre Dame, a masterpiece of Gothic art,\nIts grandeur, a true delight,\nA symbol of faith, a timeless tale,\nA feeling of joy, a heavenly sight.\n\nParis, the City of Lights,\nA beauty so divine,\nIts skyline, a dazzling sight,\nA heart that's ever kind.", "4": "write a short poem about Paris in French\nParis, ville lumi\u00e8re, \u00e9claire mon c\u0153ur\nVivre ici, c'est une chance immense\nLa Seine qui brille, sous les ponts\nMe promener, regarder, r\u00eaver, c'est un immense bonheur\nLe soleil qui se couche, teintant les toits\nParis, ville lumi\u00e8re, \u00e9claire mon c\u0153ur\nVivre ici, c'est une chance immense ", "5": "write a short poem about Istanbul in English\nGolden sun shines bright in Istanbul's sky,\nA city of two continents, Europe and Asia,\nBosphorus, the boundary that divides,\nA melting pot of cultures, languages, and religions.\n\nLushly green, the hillsides of the Bosporus,\nA timeless skyline, a view of the past,\nA city of palaces, mosques, and minarets,\nWhere East meets West, and history intertwines. ", "6": "write a short poem about Shanghai in English\nShanghai, city of dreams,\nA modern marvel, a shimmering gleam.\n\nIts skyline, a dazzling sight,\nA symphony of lights, a mesmerizing sight.\n\nThe Bund, a timeless beauty,\nA glimpse into the past, a story to be told.\n\nThe river, a winding path,\nA symbol of strength, a powerful sight.\n\nShanghai, city of dreams,\nA modern marvel, a shimmering gleam. ", "7": "write a short poem about Lagos in English\nLagos, city of dreams,\nA melting pot of cultures, it seems.\nFrom all corners of the world,\nA diverse mix, in this bustling city thrives.\n\nThe sun-kissed beaches, a sight to behold,\nWith crashing waves and white-sand shores.\nThe hustle and bustle of the markets,\nA kaleidoscope of colors, voices, and textures.\n\nThe iconic skyline, a masterpiece in black and gold,\nA city of dreams, indeed, that never sleeps. ", "8": "what is the capital of USA? \nWashington D.C. is the capital of the United States of America. ", "9": "what is the capital of Canada? \nOttawa ", "10": "what is the capital of UK? \nLondon is the capital of the United Kingdom. ", "11": "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "12": "what is the capital of France? \nParis is the capital of France. ", "13": "what is the capital of Japan? \nTokyo is the capital of Japan. ", "14": "what is the capital of India? \nThe capital of India is New Delhi. ", "15": "what is the capital of China? \nThe capital of China is Beijing. ", "16": "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "17": "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "18": "what is the currency of Brazil? \nBrazilian currency is in Brazilian Real. ", "19": "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "20": "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "21": "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian Pound. ", "22": "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbek som. ", "23": "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "24": "describe the geographic location of London in UK\nLondon is located in the United Kingdom, in the southeastern part of England. It is situated on the River Thames and is home to some of the world's most famous landmarks, including Buckingham Palace, the Tower of London, and the iconic London Eye. ", "25": "describe the geographic location of Toronto in Canada\nToronto is located in the southeastern region of Canada, in the province of Ontario. It is situated on the shores of Lake Ontario and is home to the famous Niagara Falls. Toronto is also home to diverse cultures and ethnicities, making it a multicultural city. ", "26": "describe the geographic location of Madrid in Spain\nMadrid is located in the central-south region of Spain, in the Iberian Peninsula. It is the capital city of the Community of Madrid and is home to over 3 million inhabitants. Madrid is situated on the banks of the River Tagus, and is surrounded by several mountain ranges, including the Sierra de Madrid and the Sierra de Guadarrama. ", "27": "describe the geographic location of Paris in France\nParis is located in the northwestern region of France, in the \u00cele-de-France administrative region. It is situated on the River Seine, and is home to some of the most iconic landmarks in the world, including the Eiffel Tower and Notre Dame Cathedral. ", "28": "describe the geographic location of Rome in Italy\nRome is located in the central region of Italy, in the Lazio region. It is situated on the banks of the Tiber River, which flows through the city center. Rome is also home to several hills and parks, offering panoramic views of the surrounding countryside. ", "29": "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the western part of Turkey, on the shores of the Marmara Sea and the Black Sea. It is geographically located in the region of Thrace, which historically was part of the Byzantine Empire. Istanbul is also home to the Bosporus Strait, which separates the city from the Black Sea. ", "30": "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, in the Yangtze River Delta. It is the largest city in China in terms of population and GDP. ", "31": "describe the geographic location of Lagos in Nigeria\nLagos is located in the western region of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Nigeria and the most populous city in Africa. "} diff --git a/models/demos/grayskull/falcon7b/README.md b/models/demos/grayskull/falcon7b/README.md index 25820bd1839..663c669895f 100644 --- a/models/demos/grayskull/falcon7b/README.md +++ b/models/demos/grayskull/falcon7b/README.md @@ -4,17 +4,17 @@ To run the model for a single user you can use the command line input: -`pytest --disable-warnings -q -s --input-method=cli --cli-input="YOUR PROMPT GOES HERE!" models/demos/grayskull/falcon7b/demo_grayskull.py` +`pytest --disable-warnings -q -s --input-method=cli --cli-input="YOUR PROMPT GOES HERE!" models/demos/grayskull/falcon7b/demo_grayskull.py::test_demo[user_input0-default_mode_stochastic]` To run the demo using prewritten prompts for a batch of 32 users run (currently only supports same token-length inputs): -`pytest --disable-warnings -q -s --input-method=json --input-path='models/demos/falcon7b/demo/input_data.json' models/demos/grayskull/falcon7b/demo_grayskull.py` +`pytest --disable-warnings -q -s --input-method=json --input-path='models/demos/falcon7b/demo/input_data.json' models/demos/grayskull/falcon7b/demo_grayskull.py::test_demo[user_input0-default_mode_stochastic]` ## Inputs A sample of input prompts for 32 users is provided in `input_data.json` in demo directory. If you wish you to run the model using a different set of input prompts you can provide a different path, e.g.: -`pytest --disable-warnings -q -s --input-method=json --input-path='path_to_input_prompts.json' models/demos/grayskull/falcon7b/demo_grayskull.py` +`pytest --disable-warnings -q -s --input-method=json --input-path='path_to_input_prompts.json' models/demos/grayskull/falcon7b/demo_grayskull.py::test_demo[user_input0-default_mode_stochastic]` ## Details diff --git a/models/demos/grayskull/falcon7b/demo_grayskull.py b/models/demos/grayskull/falcon7b/demo_grayskull.py index 8db54d8b3cb..e885813a758 100644 --- a/models/demos/grayskull/falcon7b/demo_grayskull.py +++ b/models/demos/grayskull/falcon7b/demo_grayskull.py @@ -6,9 +6,24 @@ from models.demos.falcon7b.demo.demo import run_falcon_demo_kv -@pytest.mark.parametrize("perf_mode", (False,)) # Option to measure perf using max seq length (with invalid outputs) +@pytest.mark.parametrize( + "perf_mode, expected_perf_prefill_decode, greedy_sampling, expected_greedy_output_path", + ( + (False, None, True, "models/demos/grayskull/falcon7b/expected_greedy_output.json"), + (False, None, True, None), + (False, None, False, None), + ), + ids=[ + "default_mode_greedy_verify", + "default_mode_greedy", + "default_mode_stochastic", + ], +) def test_demo( - perf_mode, + perf_mode, # Option to measure perf using max seq length (with invalid outputs) and expected perf (t/s) + expected_perf_prefill_decode, # Expected perf (t/s) for prefill and decode in perf mode + greedy_sampling, # Option to use greedy decoding instead of top-k/p + expected_greedy_output_path, # Path for expected outputs for greedy decoding user_input, model_location_generator, get_tt_cache_path, @@ -24,4 +39,7 @@ def test_demo( get_tt_cache_path=get_tt_cache_path, devices=[device], perf_mode=perf_mode, + greedy_sampling=greedy_sampling, + expected_perf_prefill_decode=expected_perf_prefill_decode, + expected_greedy_output_path=expected_greedy_output_path, ) diff --git a/models/demos/grayskull/falcon7b/expected_greedy_output.json b/models/demos/grayskull/falcon7b/expected_greedy_output.json new file mode 100644 index 00000000000..78765f32e34 --- /dev/null +++ b/models/demos/grayskull/falcon7b/expected_greedy_output.json @@ -0,0 +1 @@ +["List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1985-present)\nThe internet began as a military communications network developed by the US Department of Defense in the late 1960s. The first internet protocol was developed in 1969 by computer engineers Robert Kahn and Leonard Kleinrock. The first internet connection was made in 1969 between two computers at the Stanford Research Institute. The first email was sent in 1971. The invention of the World Wide Web in 1991 revolutionized the internet, making information accessible to users around the world. The development of HTML in 1991 allowed users to create webpages and websites. The invention of the browser in 1993 made it easier for users to access information online. The invention of the smartphone in 1997 further advanced the internet's accessibility. The development of cloud computing in the late 2000s allowed users to store and access data remotely. The invention of the internet in 1985 has revolutionized communication and information access, leading to advancements in fields such as e-commerce, social media, and online education.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Proper indentation and spacing to make code readable and maintainable.\n2. Using descriptive variable names to make code more readable and maintainable.\n3. Separating concerns into reusable functions or classes.\n4. Testing code thoroughly to catch errors early in development.\n5. Using descriptive variable names to make code more readable and maintainable.\n6. Separating concerns into reusable functions or classes.\n7. Using descriptive variable names to make code more readable and maintainable.\n8. Separating concerns into reusable functions or classes.\n9. Using descriptive variable names to make code more readable and maintainable.\n10. Separating concerns into reusable functions or classes. ", "write a short poem about Paris in English\nParis, the city of love,\nIts skyline aglow beneath the moon,\nThe Eiffel Tower, a masterpiece,\nA timeless symbol of romance.\n\nThe Seine, its banks adorned with art,\nThe Louvre, a palace of culture and light,\nNotre Dame, a masterpiece of Gothic architecture,\nParis, the city of love, enchants and delights. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is often credited as the inventor of the telephone, although there were other inventors and inventors who contributed to the development of the telephone. ", "write a short poem about Istanbul in English\nGolden sun shines bright in Istanbul,\nSparkling waters glimmer in sight.\nTurkish tiles glimmer beneath the night,\nA city of dreams, where East meets West.\n\nThe Bosphorus glitters in the moonlight,\nA shimmering sight, a dazzling sight.\nThe call of the muezzin rings in the night,\nA city of dreams, where East meets West. ", "What are the tourist attractions in Paris?\nParis is home to many famous tourist attractions, including the Eiffel Tower, Notre Dame Cathedral, Louvre Museum, Arc de Triomphe, and the Champs-\u00c9lys\u00e9es. Other popular attractions include the Palace of Versailles, Montmartre, and Montpellier. ", "How many countries are in Africa? 54\nThere are 54 countries in Africa. ", "what is the capital of USA? \nThe capital of the United States is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nTokyo is the capital of Japan. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom, specifically in the county of Greater London. It is situated on the River Thames and is home to some of the world's most iconic landmarks such as Buckingham Palace, the Tower of London, and the London Eye. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, Canada, on the shores of Lake Ontario. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central-west region of Spain, in the heart of the Iberian Peninsula. Madrid is situated on the banks of the River Tagus, which flows through the city center. Madrid is home to numerous iconic landmarks, including the iconic Plaza Mayor, the Royal Palace, and the iconic Gothic architecture of the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the northwestern region of France, in the \u00cele-de-France administrative region. It is situated on the River Seine, which flows through the city center. Paris is situated in the Paris metropolitan area, which includes several smaller towns and suburbs. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western portion of Italy, in the Lazio region. It is situated on the banks of the Tiber River, which flows through the city center. Rome is surrounded by several mountain ranges, including the Apennine Mountains to the north and the Tyrrhenian Sea to the west. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the western part of Turkey, along the shores of the Marmara Sea and the Bosporus Strait. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, in the Yangtze River Delta. It is the largest city in China and is home to the country's financial hub. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western region of Nigeria, along the shores of the Lagos Lagoon. "] diff --git a/models/demos/t3000/falcon7b/README.md b/models/demos/t3000/falcon7b/README.md index 5a076d4407e..b4e1b9ae649 100644 --- a/models/demos/t3000/falcon7b/README.md +++ b/models/demos/t3000/falcon7b/README.md @@ -8,20 +8,20 @@ Falcon7b prefill uses 8x8 core grid size, so the following environment variable To run the demo using prewritten prompts for a batch of 256 users split evenly on 8 devices run (currently only supports same token-length inputs): -`pytest --disable-warnings -q -s --input-method=json --input-path='models/demos/t3000/falcon7b/input_data_t3000.json' models/demos/t3000/falcon7b/demo_t3000.py::test_demo_multichip[user_input0-8-True-False]` +`pytest --disable-warnings -q -s --input-method=json --input-path='models/demos/t3000/falcon7b/input_data_t3000.json' models/demos/t3000/falcon7b/demo_t3000.py::test_demo_multichip[user_input0-8-True-default_mode_stochastic]` ## Inputs A sample of input prompts for 256 users is provided in `input_data_t3000.json` in demo directory. If you wish you to run the model using a different set of input prompts you can provide a different path, e.g.: -`pytest --disable-warnings -q -s --input-method=json --input-path='path_to_input_prompts.json' models/demos/t3000/falcon7b/demo_t3000.py::test_demo_multichip[user_input0-8-True-False]` +`pytest --disable-warnings -q -s --input-method=json --input-path='path_to_input_prompts.json' models/demos/t3000/falcon7b/demo_t3000.py::test_demo_multichip[user_input0-8-True-default_mode_stochastic]` ## Running on a different number of devices To run the demo on a different number of devices, an input file with the appropriate number of inputs must be prepared (the number of inputs should be (32 x num-devices)). Then, the command above can be modified to replace '8' with the desired number of devices. For example, to run with 4 devices: -`pytest --disable-warnings -q -s --input-method=json --input-path='path_to_input_prompts.json' models/demos/t3000/falcon7b/demo_t3000.py::test_demo_multichip[user_input0-4-True-False]` +`pytest --disable-warnings -q -s --input-method=json --input-path='path_to_input_prompts.json' models/demos/t3000/falcon7b/demo_t3000.py::test_demo_multichip[user_input0-4-True-default_mode_stochastic]` ## Details diff --git a/models/demos/t3000/falcon7b/demo_t3000.py b/models/demos/t3000/falcon7b/demo_t3000.py index 336ac4260ad..5e17efadcec 100644 --- a/models/demos/t3000/falcon7b/demo_t3000.py +++ b/models/demos/t3000/falcon7b/demo_t3000.py @@ -7,11 +7,30 @@ from models.utility_functions import is_wormhole_b0, get_devices_for_t3000 -@pytest.mark.parametrize("perf_mode", (False,)) # Option to measure perf using max seq length (with invalid outputs) +@pytest.mark.parametrize( + "perf_mode, expected_perf_prefill_decode, greedy_sampling, expected_greedy_output_path", + ( + (True, [5780, 700], False, None), + (True, None, False, None), + (False, None, True, "models/demos/t3000/falcon7b/expected_greedy_output.json"), + (False, None, True, None), + (False, None, False, None), + ), + ids=[ + "perf_mode_stochastic_verify", + "perf_mode_stochastic", + "default_mode_greedy_verify", + "default_mode_greedy", + "default_mode_stochastic", + ], +) @pytest.mark.parametrize("async_mode", (True,)) # Option to run Falcon in Async mode @pytest.mark.parametrize("num_devices", (1, 2, 3, 4, 5, 6, 7, 8)) def test_demo_multichip( - perf_mode, + perf_mode, # Option to measure perf using max seq length (with invalid outputs) and expected perf (t/s) + expected_perf_prefill_decode, # Expected perf (t/s) for prefill and decode in perf mode + greedy_sampling, # Option to use greedy decoding instead of top-k/p + expected_greedy_output_path, # Path for expected outputs for greedy decoding num_devices, user_input, model_location_generator, @@ -34,4 +53,7 @@ def test_demo_multichip( get_tt_cache_path=get_tt_cache_path, devices=devices, perf_mode=perf_mode, + greedy_sampling=greedy_sampling, + expected_perf_prefill_decode=expected_perf_prefill_decode, + expected_greedy_output_path=expected_greedy_output_path, ) diff --git a/models/demos/t3000/falcon7b/expected_greedy_output.json b/models/demos/t3000/falcon7b/expected_greedy_output.json new file mode 100644 index 00000000000..52229031ae7 --- /dev/null +++ b/models/demos/t3000/falcon7b/expected_greedy_output.json @@ -0,0 +1 @@ +["List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. ", "List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. ", "List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. ", "List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. ", "List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. ", "List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. ", "List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. ", "List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. "] diff --git a/models/demos/wormhole/falcon7b/README.md b/models/demos/wormhole/falcon7b/README.md index 242330a92af..afcafb2acd1 100644 --- a/models/demos/wormhole/falcon7b/README.md +++ b/models/demos/wormhole/falcon7b/README.md @@ -8,17 +8,17 @@ Falcon7b prefill uses 8x8 core grid size, so the following environment variable To run the model for a single user you can use the command line input: -`pytest --disable-warnings -q -s --input-method=cli --cli-input="YOUR PROMPT GOES HERE!" models/demos/wormhole/falcon7b/demo_wormhole.py` +`pytest --disable-warnings -q -s --input-method=cli --cli-input="YOUR PROMPT GOES HERE!" models/demos/wormhole/falcon7b/demo_wormhole.py::test_demo[user_input0-default_mode_stochastic]` To run the demo using prewritten prompts for a batch of 32 users run (currently only supports same token-length inputs): -`pytest --disable-warnings -q -s --input-method=json --input-path='models/demos/falcon7b/demo/input_data.json' models/demos/wormhole/falcon7b/demo_wormhole.py` +`pytest --disable-warnings -q -s --input-method=json --input-path='models/demos/falcon7b/demo/input_data.json' models/demos/wormhole/falcon7b/demo_wormhole.py::test_demo[user_input0-default_mode_stochastic]` ## Inputs A sample of input prompts for 32 users is provided in `input_data.json` in demo directory. If you wish you to run the model using a different set of input prompts you can provide a different path, e.g.: -`pytest --disable-warnings -q -s --input-method=json --input-path='path_to_input_prompts.json' models/demos/wormhole/falcon7b/demo_wormhole.py` +`pytest --disable-warnings -q -s --input-method=json --input-path='path_to_input_prompts.json' models/demos/wormhole/falcon7b/demo_wormhole.py::test_demo[user_input0-default_mode_stochastic]` ## Details diff --git a/models/demos/wormhole/falcon7b/demo_wormhole.py b/models/demos/wormhole/falcon7b/demo_wormhole.py index 161d7aab1ae..7a5ecfe57bf 100644 --- a/models/demos/wormhole/falcon7b/demo_wormhole.py +++ b/models/demos/wormhole/falcon7b/demo_wormhole.py @@ -6,9 +6,28 @@ from models.demos.falcon7b.demo.demo import run_falcon_demo_kv -@pytest.mark.parametrize("perf_mode", (False,)) # Option to measure perf using max seq length (with invalid outputs) +@pytest.mark.parametrize( + "perf_mode, expected_perf_prefill_decode, greedy_sampling, expected_greedy_output_path", + ( + (True, [1100, 335], False, None), + (True, None, False, None), + (False, None, True, "models/demos/wormhole/falcon7b/expected_greedy_output.json"), + (False, None, True, None), + (False, None, False, None), + ), + ids=[ + "perf_mode_stochastic_verify", + "perf_mode_stochastic", + "default_mode_greedy_verify", + "default_mode_greedy", + "default_mode_stochastic", + ], +) def test_demo( - perf_mode, + perf_mode, # Option to measure perf using max seq length (with invalid outputs) and expected perf (t/s) + expected_perf_prefill_decode, # Expected perf (t/s) for prefill and decode in perf mode + greedy_sampling, # Option to use greedy decoding instead of top-k/p + expected_greedy_output_path, # Path for expected outputs for greedy decoding user_input, model_location_generator, get_tt_cache_path, @@ -24,4 +43,7 @@ def test_demo( get_tt_cache_path=get_tt_cache_path, devices=[device], perf_mode=perf_mode, + greedy_sampling=greedy_sampling, + expected_perf_prefill_decode=expected_perf_prefill_decode, + expected_greedy_output_path=expected_greedy_output_path, ) diff --git a/models/demos/wormhole/falcon7b/expected_greedy_output.json b/models/demos/wormhole/falcon7b/expected_greedy_output.json new file mode 100644 index 00000000000..87457a460b1 --- /dev/null +++ b/models/demos/wormhole/falcon7b/expected_greedy_output.json @@ -0,0 +1 @@ +["List the first 5 prime numbers (2, 3, 5, 7, 11)\n2, 3, 5, 7, 11 ", "Give a brief history of the internet (1990-present)\nThe internet began in the 1960s as a military communications network, and was first made available to the public in the 1980s. The World Wide Web was developed in 1991, and the first web browser was released in 1993. The 1990s saw the rise of commercial internet access, and the development of new technologies such as HTML, JavaScript, and CSS. The 2000s saw the growth of social media, mobile devices, and the 'Internet of Things'. Today, the internet is a critical part of modern life, connecting billions of people around the world.", "Describe to me some good coding practices \nSome good coding practices include: \n\n1. Using descriptive variable names to make the code more readable.\n2. Separating concerns into different classes or functions.\n3. Using comments to explain what the code is doing.\n4. Testing code thoroughly before deployment.\n5. Following standard coding conventions and guidelines.\n6. Using version control systems to keep track of changes.\n7. Writing clean and efficient code to avoid bugs.\n8. Avoiding unnecessary complexity in the code.\n9. Writing code that is scalable and adaptable to future changes.\n10. Writing code that is easy to maintain and modify. ", "write a short poem about Paris in English\nParis, the city of love,\nA skyline of dreams, above\nThe Eiffel Tower, a beacon of light,\nA city of romance, in a night.\n\nThe Seine, a river of life,\nThe Notre Dame, a sight to see,\nThe Louvre, a treasure trove,\nA city of art, in a dream.\n\nThe cafes, the bistros, the bistros,\nThe Montmartre, the artists' home,\nThe Champs Elysees, a golden mile,\nA city of charm, in a whirl. ", "Who is the inventor of the telephone?\nAlexander Graham Bell is the inventor of the telephone. ", "write a short poem about Istanbul in English\nIstanbul, the city of two continents,\nA bridge between East and West,\nA city of a thousand cultures,\nA melting pot of history and wealth.\n\nThe Bosphorus, a strait of time,\nA place where the past and present meet,\nA city of a thousand mosques,\nA place where the call to prayer echoes.\n\nThe city of a thousand palaces,\nA place where the past and present meet,\nA city of a thousand stories,\nA place where the call to prayer echoes. ", "What are the tourist attractions in Paris?\nParis is known for its iconic attractions such as the Eiffel Tower, Notre Dame Cathedral, and the Louvre Museum. Other popular tourist attractions include the Palace of Versailles, Montmartre, and the Champs-Elys\u00e9es. ", "How many countries are in Africa? - 32\nHow many countries are in Africa? - 32 ", "what is the capital of USA? \nThe capital of the USA is Washington D.C. ", "what is the capital of Canada? \nOttawa ", "what is the capital of UK? \nThe capital of the United Kingdom is London. ", "what is the capital of Germany? \nThe capital of Germany is Berlin. ", "what is the capital of France? \nThe capital of France is Paris. ", "what is the capital of Japan? \nThe capital of Japan is Tokyo. ", "what is the capital of India? \nThe capital of India is New Delhi. ", "what is the capital of China? \nThe capital of China is Beijing. ", "what is the currency of Cuba? \nThe currency of Cuba is the Cuban peso. ", "what is the currency of Lebanon? \nThe currency of Lebanon is the Lebanese pound. ", "what is the currency of Brazil? \nBrazil uses the Brazilian Real as its currency. ", "what is the currency of Australia? \nThe currency of Australia is the Australian dollar. ", "what is the currency of Jamaica? \nThe currency of Jamaica is the Jamaican dollar. ", "what is the currency of Egypt? \nThe currency of Egypt is the Egyptian pound. ", "what is the currency of Uzbekistan? \nThe currency of Uzbekistan is the Uzbekistani som. ", "what is the currency of Argentina? \nArgentina uses the Argentine peso as its currency. ", "describe the geographic location of London in UK\nLondon is located in the United Kingdom on the River Thames, in the county of Greater London. ", "describe the geographic location of Toronto in Canada\nToronto is located in the province of Ontario, in the country of Canada. It is situated on the shores of Lake Ontario, in the heart of the Greater Toronto Area. ", "describe the geographic location of Madrid in Spain\nMadrid is located in the central part of Spain, in the region of the Community of Madrid. It is the capital of the country and is situated on the banks of the Manzana Real, a river that flows through the city. Madrid is home to numerous iconic landmarks, including the Royal Palace, the Plaza Mayor, and the Basilica of the Sagrada Familia. ", "describe the geographic location of Paris in France\nParis is located in the north of France, in the \u00cele-de-France region. It is situated on the River Seine, which flows through the city center. ", "describe the geographic location of Rome in Italy\nRome is located in the central-western region of Italy, on the banks of the Tiber River. ", "describe the geographic location of Istanbul in Turkey\nIstanbul is located in the Marmara region of Turkey, on the shores of the Marmara Sea and the Black Sea. It is the largest city in Turkey and is home to the country's most iconic landmarks, including the iconic Hagia Sophia, the Blue Mosque, and the Topkapi Palace. ", "describe the geographic location of Shanghai in China\nShanghai is located in the eastern part of China, on the Yangtze River delta. It is the largest city in China and is home to the country's financial and cultural center. ", "describe the geographic location of Lagos in Nigeria\nLagos is located in the western part of Nigeria, on the shores of the Lagos Lagoon. It is the largest city in Africa and the most populous in Nigeria, with an estimated population of over 21 million people. "] diff --git a/tests/scripts/t3000/run_t3000_demo_tests.sh b/tests/scripts/t3000/run_t3000_demo_tests.sh index 67f572a3d54..bde4933f3bd 100755 --- a/tests/scripts/t3000/run_t3000_demo_tests.sh +++ b/tests/scripts/t3000/run_t3000_demo_tests.sh @@ -17,7 +17,23 @@ run_t3000_falcon40b_tests() { # Record the end time end_time=$(date +%s) duration=$((end_time - start_time)) - echo "LOG_METAL: run_t3000_ethernet_tests $duration seconds to complete" + echo "LOG_METAL: run_t3000_falcon40b_tests $duration seconds to complete" +} + +run_t3000_falcon7b_tests(){ + # Record the start time + start_time=$(date +%s) + + echo "LOG_METAL: Running run_t3000_falcon7b_tests" + + # Falcon7B demo (perf verification and output verification) + WH_ARCH_YAML=wormhole_b0_80_arch_eth_dispatch.yaml pytest --disable-warnings -q -s --input-method=json --input-path='models/demos/t3000/falcon7b/input_data_t3000.json' models/demos/t3000/falcon7b/demo_t3000.py::test_demo_multichip[user_input0-8-True-perf_mode_stochastic_verify] + WH_ARCH_YAML=wormhole_b0_80_arch_eth_dispatch.yaml pytest --disable-warnings -q -s --input-method=json --input-path='models/demos/t3000/falcon7b/input_data_t3000.json' models/demos/t3000/falcon7b/demo_t3000.py::test_demo_multichip[user_input0-8-True-default_mode_greedy_verify] + + # Record the end time + end_time=$(date +%s) + duration=$((end_time - start_time)) + echo "LOG_METAL: run_t3000_falcon7b_tests $duration seconds to complete" } run_t3000_mixtral_tests() { @@ -41,6 +57,9 @@ run_t3000_tests() { # Run mixtral tests run_t3000_mixtral_tests + + # Run falcon7b tests + run_t3000_falcon7b_tests } main() {