How to use miniGpt4 to interference on multi-Gpus

[qmzpg]

I change code in 'MiniGPT4-video/minigpt4/models/mini_gpt4_llama_v2.py' to let miniGpt4 can interference on multi Gpus

   if self.low_resource:
        self.llama_model = llm_model.from_pretrained(
            llama_model,
            torch_dtype=torch.float16,
            # torch_dtype = torch.bfloat16,
            load_in_8bit=True,
            device_map="auto",

        )

But the result wil be wrong:
Traceback (most recent call last):
File "/home///./.local/lib/python3.10/site-packages/gradio/routes.py", line 488, in run_predict
output = await app.get_blocks().process_api(
File "/home///./.local/lib/python3.10/site-packages/gradio/blocks.py", line 1431, in process_api
result = await self.call_function(
File "/home///./.local/lib/python3.10/site-packages/gradio/blocks.py", line 1103, in call_function
prediction = await anyio.to_thread.run_sync(
File "/home///./fixfolder001/Anaconda/python310/lib/python3.10/site-packages/anyio/to_thread.py", line 28, in run_sync
return await get_asynclib().run_sync_in_worker_thread(func, *args, cancellable=cancellable,
File "/home///./fixfolder001/Anaconda/python310/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 818, in run_sync_in_worker_thread
return await future
File "/home///./fixfolder001/Anaconda/python310/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 754, in run
result = context.run(func, *args)
File "/home///./.local/lib/python3.10/site-packages/gradio/utils.py", line 707, in wrapper
response = f(*args, kwargs)
File "/home///./dnn/test/MiniGPT4-video/minigpt4_video_demo.py", line 229, in gradio_demo_local
pred=run(video_path,instruction,model,vis_processor,gen_subtitles=has_sub)
File "/home///./dnn/test/MiniGPT4-video/minigpt4_video_demo.py", line 147, in run
answers = model.generate(prepared_images, prompt, max_new_tokens=args.max_new_tokens, do_sample=True, lengths=[length],num_beams=2)
File "/home///./.local/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, kwargs)
File "/home///./dnn/test/MiniGPT4-video/minigpt4/models/mini_gpt4_llama_v2.py", line 553, in generate
outputs = self.llama_model.generate(
File "/home///./.local/lib/python3.10/site-packages/peft/peft_model.py", line 580, in generate
return self.base_model.generate(kwargs)
File "/home///./.local/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, kwargs)
File "/home///./.local/lib/python3.10/site-packages/transformers/generation/utils.py", line 1590, in generate
return self.beam_sample(
File "/home///./.local/lib/python3.10/site-packages/transformers/generation/utils.py", line 3271, in beam_sample
outputs = self(
File "/home///./.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, kwargs)
File "/home///./.local/lib/python3.10/site-packages/accelerate/hooks.py", line 165, in new_forward
output = module._old_forward(*args, kwargs)
File "/home///./dnn/test/MiniGPT4-video/minigpt4/models/modeling_llama_v2.py", line 65, in forward
outputs = self.model(
File "/home///./.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, kwargs)
File "/home///./.local/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 1070, in forward
layer_outputs = decoder_layer(
File "/home///./.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, kwargs)
File "/home///./.local/lib/python3.10/site-packages/accelerate/hooks.py", line 165, in new_forward
output = module._old_forward(*args, kwargs)
File "/home///./.local/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 795, in forward
hidden_states = self.input_layernorm(hidden_states)
File "/home///./.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, kwargs)
File "/home///./.local/lib/python3.10/site-packages/accelerate/hooks.py", line 165, in new_forward
output = module._old_forward(*args, kwargs)
File "/home/**/.local/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 117, in forward
return self.weight * hidden_states.to(input_dtype)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cuda:1!