From 262a6f6055641d0e4cc04ab9000c9711c2f11126 Mon Sep 17 00:00:00 2001 From: minmin-intel Date: Wed, 11 Sep 2024 09:05:34 -0700 Subject: [PATCH] Agent example for v1.0 release (#684) Signed-off-by: minmin-intel Signed-off-by: Chendi.Xue --- .../intel/cpu/xeon/compose_openai.yaml | 8 +- .../cpu/xeon/launch_agent_service_openai.sh | 5 +- .../intel/hpu/gaudi/compose.yaml | 94 +++++++++++++++++++ .../gaudi/launch_agent_service_tgi_gaudi.sh | 43 +++++++++ AgentQnA/docker_image_build/build.yaml | 13 +++ AgentQnA/example_data/test_docs_music.jsonl | 27 ++++++ AgentQnA/retrieval_tool/README.md | 37 ++++++++ AgentQnA/retrieval_tool/index_data.py | 77 +++++++++++++++ .../retrieval_tool/launch_retrieval_tool.sh | 25 +++++ AgentQnA/retrieval_tool/run_ingest_data.sh | 7 ++ AgentQnA/tests/1_build_images.sh | 49 ++++++++++ AgentQnA/tests/2_start_retrieval_tool.sh | 26 +++++ .../3_ingest_data_and_validate_retrieval.sh | 68 ++++++++++++++ .../4_launch_and_validate_agent_openai.sh | 60 ++++++++++++ .../tests/4_launch_and_validate_agent_tgi.sh | 76 +++++++++++++++ .../tests/_test_compose_openai_on_xeon.sh | 81 ++++++---------- AgentQnA/tests/test_compose_on_gaudi.sh | 73 ++++++++++++++ AgentQnA/tools/worker_agent_tools.py | 27 ++++++ AgentQnA/tools/worker_agent_tools.yaml | 10 +- .../intel/cpu/xeon/compose.yaml | 4 +- .../docker_image_build/build.yaml | 8 +- 21 files changed, 753 insertions(+), 65 deletions(-) create mode 100644 AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml create mode 100644 AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_tgi_gaudi.sh create mode 100644 AgentQnA/docker_image_build/build.yaml create mode 100644 AgentQnA/example_data/test_docs_music.jsonl create mode 100644 AgentQnA/retrieval_tool/README.md create mode 100644 AgentQnA/retrieval_tool/index_data.py create mode 100644 AgentQnA/retrieval_tool/launch_retrieval_tool.sh create mode 100644 AgentQnA/retrieval_tool/run_ingest_data.sh create mode 100644 AgentQnA/tests/1_build_images.sh create mode 100644 AgentQnA/tests/2_start_retrieval_tool.sh create mode 100644 AgentQnA/tests/3_ingest_data_and_validate_retrieval.sh create mode 100644 AgentQnA/tests/4_launch_and_validate_agent_openai.sh create mode 100644 AgentQnA/tests/4_launch_and_validate_agent_tgi.sh create mode 100644 AgentQnA/tests/test_compose_on_gaudi.sh create mode 100644 AgentQnA/tools/worker_agent_tools.py diff --git a/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml b/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml index 1dc46cd743..5319952d1f 100644 --- a/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml +++ b/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml @@ -14,7 +14,7 @@ services: environment: ip_address: ${ip_address} strategy: rag_agent - recursion_limit: ${recursion_limit} + recursion_limit: ${recursion_limit_worker} llm_engine: openai OPENAI_API_KEY: ${OPENAI_API_KEY} model: ${model} @@ -23,6 +23,7 @@ services: streaming: false tools: /home/user/tools/worker_agent_tools.yaml require_human_feedback: false + RETRIEVAL_TOOL_URL: ${RETRIEVAL_TOOL_URL} no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} @@ -31,6 +32,7 @@ services: LANGCHAIN_PROJECT: "opea-worker-agent-service" port: 9095 + supervisor-react-agent: image: opea/comps-agent-langchain:latest container_name: react-agent-endpoint @@ -43,13 +45,13 @@ services: environment: ip_address: ${ip_address} strategy: react_langgraph - recursion_limit: ${recursion_limit} + recursion_limit: ${recursion_limit_supervisor} llm_engine: openai OPENAI_API_KEY: ${OPENAI_API_KEY} model: ${model} temperature: ${temperature} max_new_tokens: ${max_new_tokens} - streaming: ${streaming} + streaming: false tools: /home/user/tools/supervisor_agent_tools.yaml require_human_feedback: false no_proxy: ${no_proxy} diff --git a/AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh b/AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh index 4ae02de8c8..6c2094cc8e 100644 --- a/AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh +++ b/AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh @@ -1,13 +1,16 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ export ip_address=$(hostname -I | awk '{print $1}') -export recursion_limit=12 +export recursion_limit_worker=12 +export recursion_limit_supervisor=10 export model="gpt-4o-mini-2024-07-18" export temperature=0 export max_new_tokens=512 export OPENAI_API_KEY=${OPENAI_API_KEY} export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions" +export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool" export CRAG_SERVER=http://${ip_address}:8080 docker compose -f compose_openai.yaml up -d diff --git a/AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml new file mode 100644 index 0000000000..ba890cf34d --- /dev/null +++ b/AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -0,0 +1,94 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + tgi-server: + image: ghcr.io/huggingface/tgi-gaudi:2.0.4 + container_name: tgi-server + ports: + - "8085:80" + volumes: + - ${HF_CACHE_DIR}:/data + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + PT_HPU_ENABLE_LAZY_COLLECTIVES: true + runtime: habana + cap_add: + - SYS_NICE + ipc: host + command: --model-id ${LLM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192 --sharded true --num-shard ${NUM_SHARDS} + worker-docgrader-agent: + image: opea/comps-agent-langchain:latest + container_name: docgrader-agent-endpoint + depends_on: + - tgi-server + volumes: + # - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/ + - ${TOOLSET_PATH}:/home/user/tools/ + ports: + - "9095:9095" + ipc: host + environment: + ip_address: ${ip_address} + strategy: rag_agent + recursion_limit: ${recursion_limit_worker} + llm_engine: tgi + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + llm_endpoint_url: ${LLM_ENDPOINT_URL} + model: ${LLM_MODEL_ID} + temperature: ${temperature} + max_new_tokens: ${max_new_tokens} + streaming: false + tools: /home/user/tools/worker_agent_tools.yaml + require_human_feedback: false + RETRIEVAL_TOOL_URL: ${RETRIEVAL_TOOL_URL} + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} + LANGCHAIN_PROJECT: "opea-worker-agent-service" + port: 9095 + + + supervisor-react-agent: + image: opea/comps-agent-langchain:latest + container_name: react-agent-endpoint + depends_on: + - tgi-server + - worker-docgrader-agent + volumes: + # - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/ + - ${TOOLSET_PATH}:/home/user/tools/ + ports: + - "9090:9090" + ipc: host + environment: + ip_address: ${ip_address} + strategy: react_langgraph + recursion_limit: ${recursion_limit_supervisor} + llm_engine: tgi + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + llm_endpoint_url: ${LLM_ENDPOINT_URL} + model: ${LLM_MODEL_ID} + temperature: ${temperature} + max_new_tokens: ${max_new_tokens} + streaming: false + tools: /home/user/tools/supervisor_agent_tools.yaml + require_human_feedback: false + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2} + LANGCHAIN_PROJECT: "opea-supervisor-agent-service" + CRAG_SERVER: $CRAG_SERVER + WORKER_AGENT_URL: $WORKER_AGENT_URL + port: 9090 diff --git a/AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_tgi_gaudi.sh b/AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_tgi_gaudi.sh new file mode 100644 index 0000000000..f4154fb229 --- /dev/null +++ b/AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_tgi_gaudi.sh @@ -0,0 +1,43 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +WORKPATH=$(dirname "$PWD")/.. +# export WORKDIR=$WORKPATH/../../ +echo "WORKDIR=${WORKDIR}" +export ip_address=$(hostname -I | awk '{print $1}') +export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + +# LLM related environment variables +export HF_CACHE_DIR=${HF_CACHE_DIR} +ls $HF_CACHE_DIR +export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-70B-Instruct" +export NUM_SHARDS=4 +export LLM_ENDPOINT_URL="http://${ip_address}:8085" +export temperature=0.01 +export max_new_tokens=512 + +# agent related environment variables +export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ +echo "TOOLSET_PATH=${TOOLSET_PATH}" +export recursion_limit_worker=12 +export recursion_limit_supervisor=10 +export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions" +export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool" +export CRAG_SERVER=http://${ip_address}:8080 + +docker compose -f compose.yaml up -d + +sleep 5s +echo "Waiting tgi gaudi ready" +n=0 +until [[ "$n" -ge 100 ]] || [[ $ready == true ]]; do + docker logs tgi-server &> tgi-gaudi-service.log + n=$((n+1)) + if grep -q Connected tgi-gaudi-service.log; then + break + fi + sleep 5s +done +sleep 5s +echo "Service started successfully" diff --git a/AgentQnA/docker_image_build/build.yaml b/AgentQnA/docker_image_build/build.yaml new file mode 100644 index 0000000000..1522660190 --- /dev/null +++ b/AgentQnA/docker_image_build/build.yaml @@ -0,0 +1,13 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + agent-endpoint: + build: + context: GenAIComps + dockerfile: comps/agent/langchain/Dockerfile + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + image: ${REGISTRY:-opea}/comps-agent-langchain:${TAG:-latest} diff --git a/AgentQnA/example_data/test_docs_music.jsonl b/AgentQnA/example_data/test_docs_music.jsonl new file mode 100644 index 0000000000..2a20895152 --- /dev/null +++ b/AgentQnA/example_data/test_docs_music.jsonl @@ -0,0 +1,27 @@ +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Thriller (song) - Wikipedia\nJump to content\nMain menu\nMain menu\nmove to sidebar\nhide\nNavigation\nMain pageContentsCurrent eventsRandom articleAbout WikipediaContact usDonate\nContribute\nHelpLearn to editCommunity portalRecent changesUpload file\nSearch\nSearch\nCreate account\nLog in\nPersonal tools\nCreate account Log in\nPages for logged out editors learn more\nContributionsTalk\nContents\nmove to sidebar\nhide\n(Top)\n1Composition\n2Writing\n3Recording\n4Release\n5Music video\n6Chart performance\n7Critical reception\n8Personnel\n9Charts\nToggle Charts subsection\n9.1Weekly charts\n9.2Year-end charts\n10Certifications\n11See also\n12References\nToggle the table of contents\nThriller (song)\n33 languages\n\u0627\u0644\u0639\u0631\u0628\u064a\u0629Az\u0259rbaycancaDanskDeutsch\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03acEspa\u00f1ol\u0641\u0627\u0631\u0633\u06ccFran\u00e7aisGalego\ud55c\uad6d\uc5b4HrvatskiItaliano\u05e2\u05d1\u05e8\u05d9\u05ea\u10e5\u10d0\u10e0\u10d7\u10e3\u10da\u10d8KiswahiliMagyar\u0d2e\u0d32\u0d2f\u0d3e\u0d33\u0d02Nederlands\u65e5\u672c\u8a9eNorsk bokm\u00e5lPolskiPortugu\u00eas\u0420\u0443\u0441\u0441\u043a\u0438\u0439ShqipSimple English\u0421\u0440\u043f\u0441\u043a\u0438 / srpskiSuomiSvenska\u0ba4\u0bae\u0bbf\u0bb4\u0bcd\u0e44\u0e17\u0e22T\u00fcrk\u00e7e\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430Ti\u1ebfng Vi\u1ec7t\nEdit links\nArticleTalk\nEnglish\nReadEditView history\nTools\nTools\nmove to sidebar\nhide\nActions\nReadEditView history\nGeneral\nWhat links hereRelated changesUpload fileSpecial pagesPermanent linkPage informationCite this pageGet shortened URLDownload QR codeWikidata item\nPrint/export\nDownload as PDFPrintable version\nFrom Wikipedia, the free encyclopedia\n1983 single by Michael Jackson\nFor other songs, see Thriller (disambiguation) \u00a7\u00a0Music.\n\"Thriller\"US 12-inch singleSingle by Michael Jacksonfrom the album\nThriller B-side\"Things I Do for You\"Released\nNovember\u00a01983\u00a0(1983-11) (UK)[1]\nJanuary\u00a023,\u00a01984\u00a0(1984-01-23) (US)[2]\nRecorded1982StudioWestlake (Los Angeles, California)Genre\nDisco\nfunk\nLength\n5:57 (album version)\n4:37 (special edit)\n4:05 (remixed short version)\n5:04 (\"Starlight\" version)\nLabelEpicSongwriter(s)Rod TempertonProducer(s)Quincy JonesMichael Jackson singles chronology\n\"Say Say Say\" (1983)\n\"Thriller\" (1983)\n\"Farewell My Summer Love\" (1984)\nMusic video\"Thriller\" on YouTube"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Recorded1982StudioWestlake (Los Angeles, California)Genre\nDisco\nfunk\nLength\n5:57 (album version)\n4:37 (special edit)\n4:05 (remixed short version)\n5:04 (\"Starlight\" version)\nLabelEpicSongwriter(s)Rod TempertonProducer(s)Quincy JonesMichael Jackson singles chronology\n\"Say Say Say\" (1983)\n\"Thriller\" (1983)\n\"Farewell My Summer Love\" (1984)\nMusic video\"Thriller\" on YouTube\n\"Thriller\" is a song by the American singer Michael Jackson. It was released by Epic Records in November 1983 in the UK and on January 23, 1984, in the US, as the seventh and final single from his sixth studio album, Thriller.[3]\n\"Thriller\" is a funk song featuring a repeating synthesizer bassline and lyrics and sound effects evoking horror films. It ends with a spoken-word sequence performed by the horror actor Vincent Price. It was produced by Quincy Jones and written by Rod Temperton, who wanted to write a theatrical song to suit Jackson's love of film.\nJackson decided to release \"Thriller\" as a single after Thriller left the top of the Billboard 200 chart. The\n\"Thriller\" music video, directed by John Landis, has Jackson dancing with a horde of zombies. It has been named the greatest music video of all time by various publications and readers' polls, and doubled sales of Thriller, helping it become the best-selling album in history."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Jackson decided to release \"Thriller\" as a single after Thriller left the top of the Billboard 200 chart. The\n\"Thriller\" music video, directed by John Landis, has Jackson dancing with a horde of zombies. It has been named the greatest music video of all time by various publications and readers' polls, and doubled sales of Thriller, helping it become the best-selling album in history.\nIt was the album's seventh top-ten single on the Billboard Hot 100, reaching number four. It reached number one in Belgium, France and Spain, and the top ten in many other countries. In the week of Jackson's death in 2009, it was Jackson's bestselling track in the US, with sales of 167,000 copies on the Billboard Hot Digital Tracks chart. It entered the Billboard Hot Digital Singles Chart at number two, and remained in the charts' top ten for three consecutive weeks. \"Thriller\" is certified Diamond by the Recording Industry Association of America. It appears on several of Jackson's greatest-hits albums and has been covered by numerous artists. The song has returned to the Billboard Hot 100 chart multiple times due to its popularity around Halloween.\nComposition[edit]\n\"Thriller\"\nJackson's song \"Thriller\", released as a single in 1984; Nelson George wrote that it uses cinematic sound effects, horror film motifs, and vocal trickery to convey a sense of danger.[4]\nProblems playing this file? See media help.\n\"Thriller\" is a disco-funk song[5] The introduction features sound effects such as a creaking door, thunder, feet walking on wooden planks, winds and howling wolves.[6]\nWriting[edit]\nHorror actor Vincent Price provided the spoken-word sequence at the end of \"Thriller\"."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Problems playing this file? See media help.\n\"Thriller\" is a disco-funk song[5] The introduction features sound effects such as a creaking door, thunder, feet walking on wooden planks, winds and howling wolves.[6]\nWriting[edit]\nHorror actor Vincent Price provided the spoken-word sequence at the end of \"Thriller\".\n\"Thriller\" was written by the English songwriter Rod Temperton, who had previously written \"Rock with You\" and \"Off the Wall\" for Jackson's 1979 album Off the Wall.[7] Temperton wanted to write something theatrical to suit Jackson's love of film.[8] He improvised with bass and drum patterns until he developed the bassline that runs through the song, then wrote a chord progression that built to a climax.[8] He recalled: \"I wanted it to build and build \u2013 a bit like stretching an elastic band throughout the tune to heighten suspense.\"[8]\nTemperton's first version was titled \"Starlight\", with the chorus lyric: \"Give me some starlight / Starlight sun\".[9] The production team, led by Quincy Jones, felt the song should be the title track, but that \"Starlight\" was not a strong album title. Instead, they wanted something \"mysterious\" to match Jackson's \"evolving persona\".[8] Temperton considered several titles, including \"Midnight Man\", which Jones felt was \"going in the right direction\". Finally, he conceived \"Thriller\", but worried that it was \"a crap word to sing ... It sounded terrible! However, we got Michael to spit it into the microphone a few times and it worked.\"[8]\nWith the title decided, Temperton wrote lyrics within \"a couple of hours\".[8] He envisioned a spoken-word sequence for the ending, but did not know what form it should take. It was decided to have a famous voice from the horror genre perform it, and Jones' then-wife, Peggy Lipton, suggested her friend Vincent Price.[6] Temperton composed the words for Price's part in a taxi on the way to the studio on the day of recording.[6]\nRecording[edit]\nQuincy Jones produced \"Thriller\"."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Recording[edit]\nQuincy Jones produced \"Thriller\".\nAlong with the rest of the album, \"Thriller\" was recorded over eight weeks in 1982.[10] It was recorded at Westlake Recording Studios on Santa Monica Boulevard in Los Angeles, California.[6] The engineer Bruce Swedien had Jackson record his vocals in different approaches, doubling takes and recording at different distances from the microphone. Some background vocals were recorded in the Westlake shower stall.[6]\nThe bassline was performed on an ARP 2600 synthesizer, and the verse pads were performed on a Roland Jupiter-8 layered with a Sequential Circuits Prophet 5 and a Yamaha CS-80.[11] The percussion was created with a LinnDrum drum machine modified with sound chips from two other drum machines: a snare hi-hat and congas from an LM-1 and a clap from a TR-808. \"Thriller\" also features Rhodes piano performed by Greg Phillinganes and guitar performed by David Williams.[12]\nTo record the wolf howls, Swedien set up tape recorders up around his Great Dane in a barn overnight, but the dog never howled. Instead, Jackson recorded the howls himself.[13] For the creaking doors, Swedien rented doors designed for sound effects from the Universal Studios Lot and recorded the hinges.[13] Price recorded his part in two takes; Jones, acknowledging that doing a voice-over for a song is difficult, praised Price and described his takes as \"fabulous\".[6]\nRelease[edit]\nThe album Thriller was released in November 1982 on Epic Records and spent months at the top of the Billboard 200.[14] \"Thriller\" was not initially planned for release as a single, as Epic saw it as a novelty song.[15] The Epic executive Walter Yetnikoff asked: \"Who wants a single about monsters?\"[14]"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Release[edit]\nThe album Thriller was released in November 1982 on Epic Records and spent months at the top of the Billboard 200.[14] \"Thriller\" was not initially planned for release as a single, as Epic saw it as a novelty song.[15] The Epic executive Walter Yetnikoff asked: \"Who wants a single about monsters?\"[14]\nBy mid-1983, sales of the album had begun to decline. Jackson, who was \"obsessive\" about his sales figures,[14] urged Yetnikoff and another Epic executive, Larry Stessel, to help conceive a plan to return the album to the top of the charts. Jackson's manager Frank DiLeo suggested releasing \"Thriller\", backed by a new music video.[14][16] It was the final single from the album, released in January 1984.[15]\nAlternative versions of \"Thriller\", including the \"Starlight\" demo, were released on the anniversary reissue Thriller 40 (2022).[17]\nMusic video[edit]\nMain article: Michael Jackson's Thriller (music video)\nThe music video for \"Thriller\" references numerous horror films,[14] and stars Jackson performing a dance routine with a horde of the undead.[14] It was directed by the horror director John Landis and written by Landis and Jackson. Jackson contacted Landis after seeing his film An American Werewolf in London. The pair conceived a 13-minute short film with a budget much larger than previous music videos. Jackson's record company refused to finance it, believing Thriller had peaked, so a making-of documentary, Making Michael Jackson's Thriller, was produced to receive financing from television networks.[14]\nMichael Jackson's Thriller premiered on MTV on December 2, 1983.[18] It was launched to great anticipation and played regularly on MTV.[18]\nIt doubled sales of Thriller, and the documentary sold over a million copies, becoming the best-selling videotape at the time.[14] It is credited for transforming music videos into a serious art form, breaking down racial barriers in popular entertainment, and popularizing the making-of documentary format.[19]"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "It doubled sales of Thriller, and the documentary sold over a million copies, becoming the best-selling videotape at the time.[14] It is credited for transforming music videos into a serious art form, breaking down racial barriers in popular entertainment, and popularizing the making-of documentary format.[19]\nMany elements have had a lasting impact on popular culture, such as the zombie dance and Jackson's red jacket, designed by Landis' wife Deborah Nadoolman.[19] Fans worldwide re-enact its zombie dance and it remains popular on YouTube. The Library of Congress described it as \"the most famous music video of all time\". In 2009, it became the first music video inducted into the National Film Registry as \"culturally, historically or aesthetically\" significant.[14]\nChart performance[edit]\n\"Thriller\" entered the Billboard Hot 100 charts at number 20.[20] It reached number seven the following week,[21] number five the next, and peaked the next week at number four, where it stayed for two weeks.[22][23] It finished as the #78 single on Billboard's Hot 100 for the 1984.[24]\n\"Thriller\" charted at number 19 on the Hot R&B/Hip-Hop Songs Chart.[25] On March 10, 1984, it reached its peak at number 3.[26] \"Thriller\" debuted on the UK Singles Chart on November 19, 1983, at number 24, and the following week peaked at number ten; it appeared on the chart for 52 weeks.[27] Beginning on February 5, 1984, \"Thriller\" peaked on the French Singles Chart at number one and topped the chart for four consecutive weeks.[28] \"Thriller\" also topped the Belgian VRT Top 30 Chart for two weeks in January 1984.[29]"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Following Jackson's death in 2009, his music surged in popularity.[30] In the week of his death, \"Thriller\" was Jackson's best-selling track in the US, with sales of 167,000 copies on the Billboard Hot Digital Singles Chart.[30] On July 11, 2009, \"Thriller\" charted on the Billboard Hot Digital Singles Chart at number two (its peak), and the song remained in the charts' top ten for three consecutive weeks.[31] In the United Kingdom, the song charted at number 23 the week of Jackson's death.[32] The following week, the song reached its peak at number 12 on the UK Single Chart.[27] On July 12, 2009, \"Thriller\" peaked at number two on the Italian Singles Chart[33] and was later certified gold by the Federation of the Italian Music Industry.[34] \"Thriller\" reached at number three on the Australian ARIA Chart and Swiss Singles Chart and topped the Spanish Singles Charts for one week.[35] The song also placed within the top ten on the German Singles Chart, Norwegian Singles Chart and Irish Singles Chart, at number nine, number seven and number eight respectively.[35] \"Thriller\" also landed at number 25 on the Danish Singles Chart.[36] In the third week of July \"Thriller\" peaked at number 11 in Finland.[37]\n\"Thriller\"\nhas returned to the Billboard Hot 100 chart multiple times due to its popularity around Halloween. It re-entered the Billboard Hot 100 in October 2013 at number 42,[38] number 31 in November 2018,[39] and number 19 in November 2021, its highest placement since 1984.[40] This gave Jackson at least one top-20 hit across seven consecutive decades from 1969 on the Billboard Hot 100.[40]"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "\"Thriller\"\nhas returned to the Billboard Hot 100 chart multiple times due to its popularity around Halloween. It re-entered the Billboard Hot 100 in October 2013 at number 42,[38] number 31 in November 2018,[39] and number 19 in November 2021, its highest placement since 1984.[40] This gave Jackson at least one top-20 hit across seven consecutive decades from 1969 on the Billboard Hot 100.[40]\n\"Thriller\" was certified platinum by the Recording Industry Association of America on December 4, 1989, for sales of over one million physical units in the US[41][42] As of August 2016, the song had sold 4,024,398 copies in the US.[43] The song was later certified Diamond by RIAA for sales over 10 million equivalent-units.[44][45] \"Thriller\" reached number one on three different Billboard charts the week of November 8, 2023, more than a decade after Jackson's death. Those charts included: R&B/Hip-Hop Streaming Songs, R&B Streaming Songs and R&B Digital Song Sales charts.[46]\nCritical reception[edit]\nAshley Lasimone, of AOL's Spinner.com, noted that it \"became a signature for Jackson\" and described \"the groove of its bassline, paired with Michael's killer vocals and sleek moves\" as having \"produced a frighteningly great single.\"[47] Jon Pareles of The New York Times noted that \"'Billie Jean', 'Beat It', 'Wanna Be Startin' Somethin' ' and \"the movie in the song 'Thriller'\", were the songs, unlike the \"fluff\" \"P.Y.T.\", that were \"the hits that made Thriller a world-beater; along with Mr. Jackson's stage and video presence, listeners must have identified with his willingness to admit terror.\"[48] Ann Powers of the Los Angeles Times described \"Thriller\" as \"adequately groovy\" with a \"funked-out beat\" and lyrics \"seemingly lifted from some little kid's 'scary storybook'\".[49][50]\nPersonnel[edit]\nWritten and composed by Rod Temperton\nProduced by Quincy Jones\nMichael Jackson: lead and background vocals, LinnDrum drum machine\nRod Temperton and Brian Banks: synthesizers"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Personnel[edit]\nWritten and composed by Rod Temperton\nProduced by Quincy Jones\nMichael Jackson: lead and background vocals, LinnDrum drum machine\nRod Temperton and Brian Banks: synthesizers\nGreg Phillinganes: synthesizers, Rhodes piano\nAnthony Marinelli: synthesizer programming\nDavid Williams: guitar\nJerry Hey, Gary Grant: trumpets, flugelhorns\nLarry Williams: saxophone, flute\nBill Reichenbach: trombone\nVocal, rhythm and synthesizer arrangement by Rod Temperton\nHorn arrangement by Jerry Hey\nEffects by Bruce Cannon and Bruce Swedien\nFeaturing: Narration by Vincent Price (Not featured on original edited single version)\nCharts[edit]\nWeekly charts[edit]\nChart (1983\u20131985)\nPeakposition\nAustralia (Kent Music Report)[51]\n4\nBelgium (Ultratop 50 Flanders)[52]\n1\nCanadian RPM Top Singles[53]\n3\nFinland (Suomen virallinen singlelista)[54]\n7\nFinland Jukebox (Suomen virallinen singlelista)[54]\n3\nFrance (SNEP)[28]\n1\nIreland (IRMA)[55]\n4\nNetherlands (Dutch Top 40)[56]\n3\nNetherlands (Single Top 100)[57]\n4\nNew Zealand (Recorded Music NZ)[58]\n6\nPortugal (AFP)[59]\n1\nSouth Africa (Springbok)[60]\n26\nSpain (AFYVE)[61]\n1\nUK Singles (OCC)[27]\n10\nUS Cashbox[62]\n4\nUS Billboard Hot 100[63]\n4\nUS Billboard Hot Black Singles[64][26]\n3\nUS Billboard Adult Contemporary[65]\n24\nUS Billboard Album Rock Tracks[64][26]\n42\nUS Radio & Records CHR/Pop Airplay Chart[66]\n1\nWest Germany (Official German Charts)[67]\n9\nChart (2006)\nPeakposition\nFrance (SNEP)[68]\n35\nGermany (Media Control Charts)[35]\n9\nIreland (IRMA)[55]\n8\nItaly (FIMI)[69]\n5\nNetherlands (Single Top 100)[57]\n34\nSpain (PROMUSICAE)[35]\n1\nSwitzerland (Schweizer Hitparade)[35]\n3\nChart (2007)\nPeakposition\nSpain (PROMUSICAE)[70]\n20\nUK Singles (OCC)[27]\n57\nChart (2008)\nPeakposition\nAustria (\u00d63 Austria Top 40)[71]\n55\nNorway (VG-lista)[72]\n13\nSwitzerland (Schweizer Hitparade)[73]\n53\nUK Singles (OCC)[27]\n35\nChart (2009)\nPeakposition\nAustralia (ARIA)[74]\n3\nAustria (\u00d63 Austria Top 40)[71]\n5\nBelgium (Ultratop 50 Back Catalogue Singles Flanders)[75]\n3"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "3\nChart (2007)\nPeakposition\nSpain (PROMUSICAE)[70]\n20\nUK Singles (OCC)[27]\n57\nChart (2008)\nPeakposition\nAustria (\u00d63 Austria Top 40)[71]\n55\nNorway (VG-lista)[72]\n13\nSwitzerland (Schweizer Hitparade)[73]\n53\nUK Singles (OCC)[27]\n35\nChart (2009)\nPeakposition\nAustralia (ARIA)[74]\n3\nAustria (\u00d63 Austria Top 40)[71]\n5\nBelgium (Ultratop 50 Back Catalogue Singles Flanders)[75]\n3\nBelgium (Ultratop 30 Back Catalogue Singles Wallonia)[76]\n2\nDenmark (Tracklisten)[36]\n25\nEurope (European Hot 100 Singles)[77]\n16\nFinland (Suomen virallinen lista)[78]\n11\nFrance (SNEP)[79]\n3\nIreland (IRMA)[35]\n8\nItaly (FIMI)[69]\n2\nJapan Singles Top 100 (Oricon)[35]\n41\nNetherlands (Single Top 100)[57]\n9\nNew Zealand (RIANZ)[35]\n12\nNorway (VG-lista)[72]\n7\nSpain (PROMUSICAE)[70]\n1\nSweden (Sverigetopplistan)[80]\n10\nSwitzerland (Schweizer Hitparade)[73]\n3\nUK Singles (OCC)[27]\n12\nUS Digital Song Sales (Billboard)[81]\n2\nChart (2010)\nPeakposition\nSpain (PROMUSICAE)[70]\n12\nSwitzerland (Schweizer Hitparade)[73]\n68\nUK Singles (OCC)[27]\n68\nChart (2012)\nPeakposition\nFrance (SNEP)[68]\n143\nIreland (IRMA)[55]\n30\nUK Singles (OCC)[27]\n49\nChart (2013)\nPeakposition\nFrance (SNEP)[68]\n159\nUK Singles (OCC)[27]\n48\nUS Billboard Hot 100[82]\n42\nChart (2014)\nPeakposition\nFrance (SNEP)[68]\n152\nSpain (PROMUSICAE)[70]\n38\nUK Singles (OCC)[27]\n57\nUS Billboard Hot 100[83]\n35\nChart (2015)\nPeakposition\nFrance (SNEP)[68]\n145\nSpain (PROMUSICAE)[70]\n48\nUK Singles (OCC)[27]\n61\nUS Billboard Hot 100[84]\n45\nChart (2016)\nPeakposition\nFrance (SNEP)[68]\n164\nUK Singles (OCC)[27]\n62\nChart (2017)\nPeakposition\nFrance (SNEP)[68]\n46\nSpain (PROMUSICAE)[70]\n32\nUK Singles (OCC)[27]\n34\nChart (2018)\nPeakposition\nCanada (Canadian Hot 100)[85]\n25\nUK Singles (OCC)[27]\n63\nUS Billboard Hot 100[86][87]\n31\nChart (2019)\nPeakposition\nUS Billboard Hot 100[88]\n44\nChart (2020)\nPeakposition\nGlobal 200[89]\n51\nUK Singles (OCC)[27]\n57\nUS Billboard Hot 100[90]\n48\nChart (2021)\nPeakposition\nCanada (Canadian Hot 100)[91]\n16\nGlobal 200 (Billboard)[92]\n28\nUK Singles (OCC)[93]\n40"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "34\nChart (2018)\nPeakposition\nCanada (Canadian Hot 100)[85]\n25\nUK Singles (OCC)[27]\n63\nUS Billboard Hot 100[86][87]\n31\nChart (2019)\nPeakposition\nUS Billboard Hot 100[88]\n44\nChart (2020)\nPeakposition\nGlobal 200[89]\n51\nUK Singles (OCC)[27]\n57\nUS Billboard Hot 100[90]\n48\nChart (2021)\nPeakposition\nCanada (Canadian Hot 100)[91]\n16\nGlobal 200 (Billboard)[92]\n28\nUK Singles (OCC)[93]\n40\nUK Hip Hop/R&B (OCC)[94]\n3\nUS Billboard Hot 100[95][96]\n19\nUS Billboard Digital Songs Sales[97]\n9\nChart (2022)\nPeakposition\nCanada (Canadian Hot 100)[98]\n25\nGlobal 200[99]\n37\nUK Singles (OCC)[27]\n41\nUS Billboard Hot 100[100]\n26\nChart (2023)\nPeakposition\nCanada (Canadian Hot 100)[101]\n22\nGlobal 200[102]\n39\nUK Singles (OCC)[103]\n20\nUS Billboard Hot 100[104]\n21\nYear-end charts[edit]\nChart (1984)\nPosition\nAustralia (Kent Music Report)[105]\n17\nBelgium (Ultratop Flanders)[106]\n26\nUS Billboard Hot 100[24]\n78\nChart (2009)\nPosition\nSweden (Sverigetopplistan)[107]\n88\nSwitzerland (Schweizer Hitparade)[108]\n81\nUK Singles (Official Charts Company)[109]\n143\nCertifications[edit]\nRegion\nCertification\nCertified units/sales\nAustralia (ARIA)[110]\n6\u00d7 Platinum\n420,000\u2021\nDenmark (IFPI Danmark)[111]\nPlatinum\n90,000\u2021\nFrance (SNEP)[112]\nPlatinum\n1,000,000*\nGermany (BVMI)[113]\nGold\n250,000\u2021\nItaly (FIMI)[114]\nPlatinum\n30,000\u2021\nJapan (RIAJ)[115] Full-length ringtone\nPlatinum\n250,000*\nMexico (AMPROFON)[116]\n4\u00d7 Platinum+Gold\n270,000\u2021\nSpain (PROMUSICAE)[117]\n2\u00d7 Platinum\n100,000*\nUnited Kingdom (BPI)[118] Digital sales since 2004\n2\u00d7 Platinum\n1,200,000\u2021\nUnited Kingdom (BPI)[119] other release\nGold\n500,000\u2021\nUnited States (RIAA)[120]\nDiamond\n10,000,000\u2021\nUnited States (RIAA)[121] Mastertone\nGold\n500,000*\n* Sales figures based on certification alone.\u2021 Sales+streaming figures based on certification alone.\nSee also[edit]\nList of best-selling singles\nList of best-selling singles in the United States\nList of most expensive music videos\nMichael Jackson's Thriller\nThriller (viral video)\nThrill the World\nReferences[edit]"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Diamond\n10,000,000\u2021\nUnited States (RIAA)[121] Mastertone\nGold\n500,000*\n* Sales figures based on certification alone.\u2021 Sales+streaming figures based on certification alone.\nSee also[edit]\nList of best-selling singles\nList of best-selling singles in the United States\nList of most expensive music videos\nMichael Jackson's Thriller\nThriller (viral video)\nThrill the World\nReferences[edit]\n^ \"New Singles (for the week ending November 11, 1983)\" (PDF). Music Week: 30. November 5, 1983.\n^ Semigran, Aly (February 7, 2011). \"Michael Jackson's 'Thriller': Story Behind the 'Glee' Cover\". MTV. Retrieved September 17, 2023.\n^ McPhate, Tim (November 2, 2017). \"Michael Jackson's \"Thriller\": For The Record\". The Recording Academy. Retrieved November 17, 2019.\n^ George 2004, p.\u00a023.\n^ Jones, Jel D. Lewis (2005). Michael Jackson, the King of Pop: The Big Picture \u2013 The Music! The Man! The Legend! The Interviews: An Anthology. Amber Books Publishing. p.\u00a06. ISBN\u00a00-9749779-0-X. Retrieved July 22, 2010.\n^ a b c d e f Lyle, Peter (November 25, 2007). \"Michael Jackson's monster smash\". The Daily Telegraph. Archived from the original on January 12, 2022. Retrieved January 24, 2010.\n^ Kreps, Daniel (October 5, 2016). \"Rod Temperton, 'Thriller' songwriter, dead at 66\". Rolling Stone. Retrieved July 25, 2022.\n^ a b c d e f \"Revealed: the story behind Jacko's Thriller\". M magazine. October 31, 2012. Archived from the original on November 3, 2012. Retrieved October 24, 2018.\n^ Glazer, Eliot (September 25, 2009). \"Top 1984 Songs\". AOLRadioBlog.com. AOL Inc. Retrieved January 24, 2010.\n^ Vozick-Levinson, Simon (February 18, 2008). \"Quincy Jones' 'Thriller' Memories\". EW.com. Retrieved January 24, 2010.\n^ Carr, Dan (November 30, 2022). \"The synth sounds of Michael Jackson's Thriller (and how to recreate them in your DAW)\". MusicRadar. Retrieved March 19, 2023."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "^ Glazer, Eliot (September 25, 2009). \"Top 1984 Songs\". AOLRadioBlog.com. AOL Inc. Retrieved January 24, 2010.\n^ Vozick-Levinson, Simon (February 18, 2008). \"Quincy Jones' 'Thriller' Memories\". EW.com. Retrieved January 24, 2010.\n^ Carr, Dan (November 30, 2022). \"The synth sounds of Michael Jackson's Thriller (and how to recreate them in your DAW)\". MusicRadar. Retrieved March 19, 2023.\n^ Rogerson, Ben (February 10, 2023). \"Watch Greg Phillinganes recreate Michael Jackson's Thriller using the original synths\". MusicRadar. Retrieved March 27, 2023.\n^ a b \"The making of Michael Jackson's Thriller\". MusicRadar. Retrieved October 25, 2018.\n^ a b c d e f g h i Griffin, Nancy (July 2010). \"The \"Thriller\" Diaries\". Vanity Fair. Retrieved January 2, 2011.\n^ a b Romano, Aja (October 31, 2018). \"Michael Jackson's \"Thriller\" is the eternal Halloween bop \u2014 and so much more\". Vox. Retrieved October 25, 2021.\n^ Eagan, Daniel (November 24, 2011). America's Film Legacy, 2009\u20132010: A Viewer's Guide to the 50 Landmark Movies Added To The National Film Registry in 2009\u201310. Bloomsbury Publishing. p.\u00a0175. ISBN\u00a0978-1-4411-9328-5. Retrieved May 14, 2016.\n^ Miles Marshall Lewis (November 30, 2022). \"#Thriller40: Cultural Critics Celebrate Michael Jackson's Impact\". BET.\n^ a b Richin, Leslie (December 2, 2014). \"On This Day In 1983, Michael Jackson's 'Thriller' Premiered On MTV\". Billboard.\n^ a b Hebblethwaite, Phil (November 21, 2013). \"How Michael Jackson's Thriller changed music videos for ever\". The Guardian. Retrieved October 29, 2018.\n^ \"Week of February 11, 1984\". Billboard. Nielsen Business Media, Inc. Retrieved October 10, 2015.\n^ \"Week of February 18, 1984\". Billboard. Nielsen Business Media, Inc. January 2, 2013. Retrieved October 10, 2015.\n^ \"Week of March 3, 1984\". Billboard. Nielsen Business Media, Inc. January 2, 2013. Retrieved October 10, 2015.\n^ \"Week of March 10, 1984\". Billboard. Nielsen Business Media, Inc. Retrieved October 10, 2015."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "^ \"Week of February 18, 1984\". Billboard. Nielsen Business Media, Inc. January 2, 2013. Retrieved October 10, 2015.\n^ \"Week of March 3, 1984\". Billboard. Nielsen Business Media, Inc. January 2, 2013. Retrieved October 10, 2015.\n^ \"Week of March 10, 1984\". Billboard. Nielsen Business Media, Inc. Retrieved October 10, 2015.\n^ a b \"Billboard Top 100 \u2013 1984\". billboard. Retrieved March 29, 2020.\n^ \"Week of March 3, 1984\". Billboard. Nielsen Business Media, Inc. Archived from the original on January 21, 2010. Retrieved January 23, 2010.\n^ a b c \"Week of March 10, 1984\". Billboard. Nielsen Business Media, Inc. Retrieved January 23, 2010.\n^ a b c d e f g h i j k l m n o p \"Michael Jackson\". Official Charts Company. Retrieved October 10, 2015.\n^ a b \"Toutes les Chansons N\u00b0 1 des Ann\u00e9es 80\". Infodisc.fr. Dominic Durand / InfoDisc. Archived from the original on November 20, 2012. Retrieved January 23, 2010.\n^ \"Michael Jackson \u2013 Thriller\". Top30-3.radio2.be (in Dutch). VRT \u2013 Auguste Reyerslaan. Archived from the original on February 22, 2012. Retrieved January 24, 2010.\n^ a b Ed Christman, Antony Bruno (July 2, 2009). \"Michael Jackson Music Sales Surge Could Last For Months\". Billboard. Nielsen Business Media, Inc. Retrieved January 23, 2010.\n^ \"July 11, 2009\". Billboard. Nielsen Business Media. Retrieved January 23, 2010.\n^ \"Chart For Week Up To 04/07/2009\". Official Charts Company. Retrieved January 23, 2010.\n^ \"Thriller in Italian Chart\". Hung Medien. Retrieved June 21, 2013.\n^ \"Certificazioni Download FIMI\" (PDF) (in Italian). Federation of the Italian Music Industry. Archived from the original (PDF) on June 5, 2012. Retrieved January 2, 2012.\n^ a b c d e f g h \"Michael Jackson \u2013 Thriller \u2013 Music Charts\". Acharts.us. Retrieved January 23, 2010.\n^ a b \"Track Top 40 \u2013 July 10, 2009\". Hitlisterne.dk. IFPI Danmark & Nielsen Music Control. Archived from the original on September 30, 2011. Retrieved January 23, 2010."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "^ a b c d e f g h \"Michael Jackson \u2013 Thriller \u2013 Music Charts\". Acharts.us. Retrieved January 23, 2010.\n^ a b \"Track Top 40 \u2013 July 10, 2009\". Hitlisterne.dk. IFPI Danmark & Nielsen Music Control. Archived from the original on September 30, 2011. Retrieved January 23, 2010.\n^ \"Thriller in Finnish Chart\". Hung Medien. Retrieved January 23, 2010.\n^ \"Eminem Debuts at No. 1 on Hot R&B/Hip-Hop Songs Chart; Michael Jackson's 'Thriller' Returns\". Billboard. November 8, 2013. Retrieved October 2, 2016.\n^ \"Top 100 Songs | Billboard Hot 100 Chart\". Billboard. Retrieved November 11, 2018.\n^ a b \"Michael Jackson's 'Thriller' Leads Halloween Treats on Billboard Hot 100\". Billboard. Retrieved November 9, 2021.\n^ \"American\nsingle\ncertifications \u2013 Michael Jackson \u2013 Thriller\". Recording Industry Association of America.\n^ \"Rock Music, etc., Terms\". Georgetown College. October 26, 1999. Archived from the original on May 18, 2011. Retrieved January 8, 2010.\n^ \"Hip Hop Single Sales: The Weeknd, Zay Hilfigerrr & Drake\". Hip Hop DX. November 13, 2016. Retrieved November 14, 2016.\n^ \"Michael Jackson's Catalogue Garners Major New Gold & Platinum Awards\". RIAA. August 23, 2018. Retrieved December 21, 2018.\n^ Appel, Rich (October 30, 2014). \"Revisionist History, Part 3: Michael Jackson Gets Revenge on Prince! Year-End Hits of the Past, Re-Analyzed\". Billboard. Prometheus Global Media. Retrieved October 30, 2014.\n^ \"Michael Jackson Scores Three No. 1 Hits On The Billboard Charts This Week\". Forbes. Retrieved November 10, 2023.\n^ Lasimone, Ashley (October 28, 2009). \"Clash of the Cover Songs: Michael Jackson vs. Imogen Heap\". Spinner.com. AOL Inc. Retrieved January 23, 2010.\n^ Pareles, Jon (September 3, 1987). \"Critic's Notebook; How Good Is Jackson's 'Bad'?\". The New York Times. Retrieved January 25, 2010.\n^ Powers, Ann (February 15, 2008). \"Nine reasons why Jackson masterpiece remains a 'Thriller'\". SouthCoastToday.com. Dow Jones Local Media Group. Retrieved February 6, 2010."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "^ Pareles, Jon (September 3, 1987). \"Critic's Notebook; How Good Is Jackson's 'Bad'?\". The New York Times. Retrieved January 25, 2010.\n^ Powers, Ann (February 15, 2008). \"Nine reasons why Jackson masterpiece remains a 'Thriller'\". SouthCoastToday.com. Dow Jones Local Media Group. Retrieved February 6, 2010.\n^ Comstock, Miriam Marcus and Courtney. \"Thriller Chiller For Jackson\". Forbes. Retrieved November 4, 2019.\n^ Kent, David (2003). Australian Chart Book 1970\u20131992. Australian Chart Book. ISBN\u00a00-646-11917-6.\n^ \"Michael Jackson \u2013 Thriller\" (in Dutch). Ultratop 50.\nRetrieved December 14, 2021.\n^ \"Top Singles \u2013 Volume 40, No. 1, March 10, 1984\". RPM. Archived from the original on October 17, 2012. Retrieved August 3, 2010.\n^ a b Pennanen, Timo (2021). \"Michael Jackson\". Sis\u00e4lt\u00e4\u00e4 hitin - 2. laitos Levyt ja esitt\u00e4j\u00e4t Suomen musiikkilistoilla 1.1.1960\u201330.6.2021 (PDF). Helsinki: Kustannusosakeyhti\u00f6 Otava. p.\u00a0113. Retrieved May 29, 2022.\n^ a b c \"Search Results: Thriller\". IrishCharts.ie. Irish Recorded Music Association. Retrieved January 25, 2010.\n^\n\"Nederlandse Top 40 \u2013 week 2, 1984\" (in Dutch). Dutch Top 40.\n^ a b c \"Michael Jackson \u2013 Thriller\" (in Dutch). Single Top 100.\n^ \"Michael Jackson \u2013 Thriller\". Top 40 Singles.\n^ \"Top 3 in Europe\" (PDF). Music & Media. May 14, 1984. p.\u00a012. Retrieved October 29, 2021.\n^ \"SA Charts 1965\u2013March 1989\". Retrieved September 5, 2018.\n^ Salaverri, Fernando (September 2005). S\u00f3lo \u00e9xitos: a\u00f1o a a\u00f1o, 1959\u20132002 (in Spanish) (1st\u00a0ed.). Spain: Fundaci\u00f3n Autor-SGAE. ISBN\u00a084-8048-639-2.\n^ Whitburn, Joel (2014). Cash Box Pop Hits 1952-1996. Sheridan Books, Inc. ISBN\u00a0978-0-89820-209-0.\n^ \"Michael Jackson Chart History (Hot 100)\". Billboard. September 9, 2021. Retrieved September 9, 2021.\n^ a b \"Allmusic (Thriller > Charts & Awards > Billboard Singles)\". Allmusic.com. Rovi Corporation. Retrieved January 23, 2010.\n^ Whitburn, Joel (1993). Top Adult Contemporary: 1961\u20131993. Record Research. p.\u00a0118.\n^ \"Michael Jackson\"."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "^ \"Michael Jackson Chart History (Hot 100)\". Billboard. September 9, 2021. Retrieved September 9, 2021.\n^ a b \"Allmusic (Thriller > Charts & Awards > Billboard Singles)\". Allmusic.com. Rovi Corporation. Retrieved January 23, 2010.\n^ Whitburn, Joel (1993). Top Adult Contemporary: 1961\u20131993. Record Research. p.\u00a0118.\n^ \"Michael Jackson\".\n^ \"Offiziellecharts.de \u2013 Michael Jackson \u2013 Thriller\" (in German). GfK Entertainment charts.\nRetrieved March 18, 2019.\n^ a b c d e f g \"Michael Jackson \u2013 Thriller\" (in French). Les classement single.\n^ a b \"Michael Jackson \u2013 Thriller\". Top Digital Download.\n^ a b c d e f \"Michael Jackson \u2013 Thriller\" Canciones Top 50.\n^ a b \"Michael Jackson \u2013 Thriller\" (in German). \u00d63 Austria Top 40.\n^ a b \"Michael Jackson \u2013 Thriller\". VG-lista.\n^ a b c \"Michael Jackson \u2013 Thriller\". Swiss Singles Chart.\n^ \"Michael Jackson \u2013 Thriller\". ARIA Top 50 Singles.\n^ \"30 Back Catalogue Singles \u2013 July 18, 2009\". UltraTop.be. Hung Medien. Retrieved January 24, 2010.\n^ \"30 Back Catalogue Singles \u2013 July 4, 2009\". UltraTop.be. Hung Medien. Retrieved January 24, 2010.\n^ \"Michael Jackson Album & Song Chart History\". Billboard.com. Nielsen Business Media. Retrieved October 29, 2011.\n^ \"Michael Jackson: Thriller\" (in Finnish). Musiikkituottajat.\n^ \"Download Single Top 50 \u2013 04/07/2009\". Lescharts.com. Hung Medien. Retrieved January 23, 2010.\n^ \"Michael Jackson \u2013 Thriller\". Singles Top 100.\n^ \"Michael Jackson Chart History (Digital Song Sales)\". Billboard.\n^ \"The Hot 100, Week of November 16, 2013\". Billboard. Prometheus Global Media. Retrieved November 12, 2015.\n^ \"The Hot 100, Week of November 15, 2014\". Billboard. Prometheus Global Media.\n^ \"The Hot 100, Week of November 21, 2015\". Billboard. Prometheus Global Media. Retrieved November 12, 2015.\n^ \"Michael Jackson Chart History (Canadian Hot 100)\". Billboard.\nRetrieved November 6, 2018.\n^ Zellner, Xander. \"Michael Jackson's 'Thriller' Returns to Hot 100, Thanks to Halloween Gains\". Billboard. Retrieved November 6, 2018."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "^ \"The Hot 100, Week of November 21, 2015\". Billboard. Prometheus Global Media. Retrieved November 12, 2015.\n^ \"Michael Jackson Chart History (Canadian Hot 100)\". Billboard.\nRetrieved November 6, 2018.\n^ Zellner, Xander. \"Michael Jackson's 'Thriller' Returns to Hot 100, Thanks to Halloween Gains\". Billboard. Retrieved November 6, 2018.\n^ \"The Hot 100, Week of November 10, 2018\". Billboard. Retrieved September 9, 2021.\n^ \"The Hot 100, Week of November 9, 2019\". Billboard. Retrieved March 20, 2021.\n^ \"The Global 200, Week of November 14, 2020\". Billboard. Retrieved November 8, 2023.\n^ \"The Hot 100, Week of November 14, 2020\". Billboard. Retrieved November 14, 2020.\n^ \"Canadian Hot 100, Week of November 13, 2021\". Billboard. Retrieved November 23, 2021.\n^ \"Michael Jackson Chart History (Global 200)\". Billboard.\nRetrieved November 9, 2021.\n^ \"Official Singles Chart Top 100\". Official Charts Company.\nRetrieved November 8, 2023.\n^ \"Official Hip Hop and R&B Singles Chart Top 40\". Official Charts Company.\nRetrieved November 5, 2021.\n^ \"Michael Jackson's 'Thriller' Leads Halloween Treats on Billboard Hot 100\". Billboard. Retrieved November 8, 2021.\n^ \"The Hot 100, Week of November 13, 2021\". Billboard. Retrieved November 5, 2022.\n^ \"Digital Song Sales Chart, Week of November 13, 2021\". Billboard. Retrieved November 11, 2021.\n^ \"Canadian Hot 100, Week of November 12, 2022\". Billboard. Retrieved November 8, 2022.\n^ \"The Global 200, Week of November 12, 2022\". Billboard. Retrieved November 8, 2022.\n^ \"The Hot 100, Week of November 12, 2022\". Billboard. Retrieved November 8, 2022.\n^ \"Canadian Hot 100, Week of November 11, 2023\". Billboard. Retrieved November 8, 2023.\n^ \"The Global 200, Week of November 11, 2023\". Billboard. Retrieved November 8, 2023.\n^ \"Official Singles Chart Top 100\". Official Charts Company.\nRetrieved November 8, 2023.\n^ \"The Hot 100, Week of November 11, 2023\". Billboard. Retrieved November 8, 2023."} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "^ \"Canadian Hot 100, Week of November 11, 2023\". Billboard. Retrieved November 8, 2023.\n^ \"The Global 200, Week of November 11, 2023\". Billboard. Retrieved November 8, 2023.\n^ \"Official Singles Chart Top 100\". Official Charts Company.\nRetrieved November 8, 2023.\n^ \"The Hot 100, Week of November 11, 2023\". Billboard. Retrieved November 8, 2023.\n^ \"Kent Music Report No 548 \u2013 31 December 1984 > National Top 100 Singles for 1984\". Kent Music Report. Retrieved January 23, 2023 \u2013 via Imgur.com.\n^ \"Jaaroverzichten 1984\". Ultratop. Retrieved December 14, 2021.\n^ \"\u00c5rslista Singular \u2013 \u00c5r 2009\" (in Swedish). Sverigetopplistan. Retrieved March 29, 2020.\n^ \"Schweizer Jahreshitparade 2009 \u2013 hitparade.ch\". Hung Medien. Retrieved March 29, 2020.\n^ \"Charts Plus Year end 2009\" (PDF). Charts Plus. Retrieved May 16, 2020.\n^ \"ARIA Charts \u2013 Accreditations \u2013 2021 Singles\" (PDF). Australian Recording Industry Association.\n^ \"Danish\nsingle\ncertifications \u2013 Michael Jackson \u2013 Thriller\". IFPI Danmark. Retrieved July 2, 2023.\n^ \"French\nsingle\ncertifications \u2013 Michael Jackson \u2013 Thriller\" (in French). InfoDisc. Retrieved November 28, 2022. Select MICHAEL JACKSON and click OK.\n^ \"Gold-/Platin-Datenbank (Michael Jackson;\u00a0'Thriller')\" (in German). Bundesverband Musikindustrie. Retrieved February 18, 2023.\n^ \"Italian\nsingle\ncertifications \u2013 Michael Jackson \u2013 Thriller\" (in Italian). Federazione Industria Musicale Italiana. Select \"2014\" in the \"Anno\" drop-down menu. Select \"Thriller\" in the \"Filtra\" field. Select \"Singoli\" under \"Sezione\".\n^ \"Japanese\nringtone\ncertifications \u2013 Michael Jackson \u2013 Thriller\" (in Japanese). Recording Industry Association of Japan. Retrieved December 30, 2020. Select 2009\u5e7411\u6708 on the drop-down menu\n^ \"Certificaciones\" (in Spanish). Asociaci\u00f3n Mexicana de Productores de Fonogramas y Videogramas. Retrieved November 28, 2022. Type Michael Jackson in the box under the ARTISTA column heading\u00a0and Thriller in the box under the T\u00cdTULO column heading.\n^ \"Spanish\nsingle"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "^ \"Certificaciones\" (in Spanish). Asociaci\u00f3n Mexicana de Productores de Fonogramas y Videogramas. Retrieved November 28, 2022. Type Michael Jackson in the box under the ARTISTA column heading\u00a0and Thriller in the box under the T\u00cdTULO column heading.\n^ \"Spanish\nsingle\ncertifications \u2013 Michael Jackson \u2013 Thriller\". El portal de M\u00fasica. Productores de M\u00fasica de Espa\u00f1a.\n^ \"British\nsingle\ncertifications \u2013 Michael Jackson \u2013 Thriller\". British Phonographic Industry. Retrieved March 17, 2023.\n^ \"British\nsingle\ncertifications \u2013 Michael Jackson \u2013 Thriller\". British Phonographic Industry. Retrieved March 17, 2023.\n^ \"American\nsingle\ncertifications \u2013 Michael Jackson \u2013 Thriller\". Recording Industry Association of America. Retrieved August 29, 2022.\n^ \"American\nringtone\ncertifications \u2013 Michael Jackson \u2013 Thriller\". Recording Industry Association of America.\nBibliography\nBrooks, Darren (2002). Michael Jackson: An Exceptional Journey. Chrome Dreams. ISBN\u00a01-84240-178-5.\nGeorge, Nelson (2004). Michael Jackson: The Ultimate Collection (booklet). Sony BMG.\nGrant, Adrian (2009). Michael Jackson: The Visual Documentary. Omnibus Press. ISBN\u00a0978-1-84938-261-8.\nJones, Jel (2005). Michael Jackson, the King of Pop: The Big Picture: the Music! the Man! the Legend! the Interviews!. Amber Books Publishing. ISBN\u00a00-9749779-0-X.\nTaraborrelli, J. Randy (2004). The Magic and the Madness. Terra Alta, WV: Headline. ISBN\u00a00-330-42005-4.\nHalstead, Craig (2003). Michael Jackson The Solo Years. On-Line Ltd. ISBN\u00a0978-0-7552-0091-7.\nvteMichael Jackson: ThrillerSide one\n\"Wanna Be Startin' Somethin'\"\n\"Baby Be Mine\"\n\"The Girl Is Mine\"\n\"Thriller\"\nSide two\n\"Beat It\"\n\"Billie Jean\"\n\"Human Nature\"\n\"P.Y.T. (Pretty Young Thing)\"\n\"The Lady in My Life\"\nRelated articles\nE.T. the Extra-Terrestrial (audiobook)\nFarewell My Summer Love\nVictory\nVictory Tour\nMichael Jackson's Thriller\nThriller jacket\nThriller 25\nThriller 40\nThriller 40 (film)\nThrill the World\nThriller viral video\nDonga\nThriller \u2013 Live"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "\"Baby Be Mine\"\n\"The Girl Is Mine\"\n\"Thriller\"\nSide two\n\"Beat It\"\n\"Billie Jean\"\n\"Human Nature\"\n\"P.Y.T. (Pretty Young Thing)\"\n\"The Lady in My Life\"\nRelated articles\nE.T. the Extra-Terrestrial (audiobook)\nFarewell My Summer Love\nVictory\nVictory Tour\nMichael Jackson's Thriller\nThriller jacket\nThriller 25\nThriller 40\nThriller 40 (film)\nThrill the World\nThriller viral video\nDonga\nThriller \u2013 Live\nMichael Jackson albums discography\nvteMichael Jackson songs\nSingles\nSongs\nUnreleased songs\n1970s\n\"Got to Be There\"\n\"Ain't No Sunshine\"\n\"I Wanna Be Where You Are\"\n\"Rockin' Robin\"\n\"Love Is Here and Now You're Gone\"\n\"You've Got a Friend\"\n\"Ben\"\n\"Everybody's Somebody's Fool\"\n\"My Girl\"\n\"Shoo-Be-Doo-Be-Doo-Da-Day\"\n\"We've Got a Good Thing Going\"\n\"With a Child's Heart\"\n\"Morning Glow\"\n\"All the Things You Are\"\n\"Happy\"\n\"Too Young\"\n\"Music and Me\"\n\"We're Almost There\"\n\"Just a Little Bit of You\"\n\"You Can't Win\"\n\"Don't Stop 'Til You Get Enough\"\n\"Rock with You\"\n\"Working Day and Night\"\n\"It's the Falling in Love\"\n1980s\n\"Off the Wall\"\n\"Girlfriend\"\n\"She's Out of My Life\"\n\"One Day in Your Life\"\n\"The Girl Is Mine\"\n\"Billie Jean\"\n\"Beat It\"\n\"Wanna Be Startin' Somethin'\"\n\"Human Nature\"\n\"P.Y.T. (Pretty Young Thing)\"\n\"Thriller\"\n\"You've Really Got a Hold on Me\"\n\"Here I Am (Come and Take Me)\"\n\"Lonely Teardrops\"\n\"That's What Love Is Made Of\"\n\"Farewell My Summer Love\"\n\"Girl You're So Together\"\n\"I Just Can't Stop Loving You\"\n\"Bad\"\n\"The Way You Make Me Feel\"\n\"Speed Demon\"\n\"Liberian Girl\"\n\"Just Good Friends\"\n\"Another Part of Me\"\n\"Man in the Mirror\"\n\"Dirty Diana\"\n\"Smooth Criminal\"\n\"Leave Me Alone\"\n\"Twenty-Five Miles\"\n1990s\n\"Black or White\"\n\"Jam\"\n\"In the Closet\"\n\"Remember the Time\"\n\"Heal the World\"\n\"Who Is It\"\n\"Give In to Me\"\n\"Will You Be There\"\n\"Gone Too Soon\"\n\"Dangerous\"\n\"Come Together\"\n\"Scream\"\n\"Childhood\"\n\"They Don't Care About Us\"\n\"Stranger in Moscow\"\n\"This Time Around\"\n\"Earth Song\"\n\"D.S.\"\n\"You Are Not Alone\"\n\"Tabloid Junkie\"\n\"HIStory\"\n\"Smile\"\n\"Blood on the Dance Floor\"\n\"Ghosts\"\n\"Is It Scary\"\n\"On the Line\"\n2000s"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "\"Jam\"\n\"In the Closet\"\n\"Remember the Time\"\n\"Heal the World\"\n\"Who Is It\"\n\"Give In to Me\"\n\"Will You Be There\"\n\"Gone Too Soon\"\n\"Dangerous\"\n\"Come Together\"\n\"Scream\"\n\"Childhood\"\n\"They Don't Care About Us\"\n\"Stranger in Moscow\"\n\"This Time Around\"\n\"Earth Song\"\n\"D.S.\"\n\"You Are Not Alone\"\n\"Tabloid Junkie\"\n\"HIStory\"\n\"Smile\"\n\"Blood on the Dance Floor\"\n\"Ghosts\"\n\"Is It Scary\"\n\"On the Line\"\n2000s\n\"Speechless\"\n\"You Rock My World\"\n\"Heaven Can Wait\"\n\"Butterflies\"\n\"Cry\"\n\"One More Chance\"\n\"Cheater\"\n\"(I Like) The Way You Love Me\"\n\"Fall Again\"\n\"This Is It\"\n2010s\n\"Hold My Hand\"\n\"Hollywood Tonight\"\n\"(I Can't Make It) Another Day\"\n\"Behind the Mask\"\n\"Don't Be Messin' 'Round\"\n\"I'm So Blue\"\n\"Price of Fame\"\n\"Love Never Felt So Good\"\n\"Chicago\"\n\"Loving You\"\n\"A Place with No Name\"\n\"Slave to the Rhythm\"\n\"Blue Gangsta\"\n2020s\n\"She's Trouble\"\nOther\n\"We Are the World\"\n\"Mind Is the Magic\"\n\"What More Can I Give\"\n\"We Are the World 25 for Haiti\"\n\"Blood on the Dance Floor x Dangerous\"\n\"Diamonds Are Invincible\"\nFeatured\n\"Ease on Down the Road\"\n\"A Brand New Day\"\n\"Night Time Lover\"\n\"Papa Was a Rollin' Stone\"\n\"State of Independence\"\n\"Muscles\"\n\"Say Say Say\"\n\"Somebody's Watching Me\"\n\"Don't Stand Another Chance\"\n\"Centipede\"\n\"Tell Me I'm Not Dreamin' (Too Good to Be True)\"\n\"Eaten Alive\"\n\"Get It\"\n\"2300 Jackson Street\"\n\"Do the Bartman\"\n\"Whatzupwitu\"\n\"Why\"\n\"I Need You\"\n\"We Be Ballin'\"\n\"Girls, Girls, Girls\"\n\"All in Your Name\"\n\"There Must Be More to Life Than This\"\n\"Low\"\n\"Don't Matter to Me\"\nCategory\nAuthority control databases\nMusicBrainz work\nRetrieved from \"https://en.wikipedia.org/w/index.php?title=Thriller_(song)&oldid=1212467768\""} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "\nCategories: 1982 songs1983 singles1984 singlesCBS Records singlesColumbia Records singlesCompositions with a narratorEpic Records singlesHalloween songsMichael Jackson songsNumber-one singles in SpainSNEP Top Singles number-one singlesSong recordings produced by Quincy JonesSongs about monstersSongs written by Rod TempertonHidden categories: CS1 Dutch-language sources (nl)CS1 Italian-language sources (it)CS1 Spanish-language sources (es)Articles with German-language sources (de)CS1 Swedish-language sources (sv)CS1 French-language sources (fr)CS1 German-language sources (de)Cite certification used for Italy without IDCS1 Japanese-language sources (ja)Articles with short descriptionShort description is different from WikidataGood articlesUse American English from November 2021All Wikipedia articles written in American EnglishUse mdy dates from November 2014Articles with hAudio microformatsCertification Cite Ref usages outside Certification Table EntrySingle chart usages for FlandersSingle chart usages for Dutch40Single chart called without artistSingle chart called without songSingle chart usages for Dutch100Single chart usages for New ZealandSingle chart usages for West GermanySingle chart usages for FranceSingle chart usages for ItalySingle chart making named refSingle chart usages for SpainSingle chart usages for AustriaSingle chart usages for NorwaySingle chart usages for SwissSingle chart usages for AustraliaSingle chart usages for FinlandSingle chart usages for SwedenSingle chart usages for BillboarddigitalsongsSingle chart usages for CanadaSingle chart usages for Billboardglobal200Single chart usages for UKSingle chart usages for UKrandbCertification Table Entry usages for AustraliaPages using certification Table Entry with streaming figuresCertification Table Entry usages for DenmarkCertification Table Entry usages for FrancePages using certification Table Entry with sales figuresCertification Table Entry usages for GermanyCertification Table Entry usages for ItalyCertification Table Entry usages for JapanCertification Table Entry usages for MexicoCertification Table Entry usages for SpainCertification Table Entry usages for United KingdomCertification Table Entry usages for United StatesPages using certification Table Entry with sales footnotePages using certification Table Entry with streaming footnoteArticles with MusicBrainz work identifiers"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "This page was last edited on 8 March 2024, at 01:12\u00a0(UTC).\nText is available under the Creative Commons Attribution-ShareAlike License 4.0;\nadditional terms may apply. By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia\u00ae is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\nPrivacy policy\nAbout Wikipedia\nDisclaimers\nContact Wikipedia\nCode of Conduct\nDevelopers\nStatistics\nCookie statement\nMobile view\nToggle limited content width"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": "Jon Pareles of The New York Times noted that "'Billie Jean', 'Beat It', 'Wanna Be Starting' Something' ' and "the movie in the song 'Thriller'", were the songs, unlike the "fluff" "P.Y.T.", that were "the hits that made Thriller a world-beater; along with Mr. Jackson's stage and video presence, ...Jon Pareles of The New York Times noted that \"'Billie Jean', 'Beat It', 'Wanna Be Starting' Something' ' and \"the movie in the song 'Thriller'\", were the songs, unlike the \"fluff\" \"P.Y.T.\", that were \"the hits that made Thriller a world-beater; along with Mr. Jackson's stage and video presence, listeners must have identified with his willingness to admit terror.\" It appears on several of Jackson's greatest-hits albums and has been covered by numerous artists. The song has returned to the Billboard Hot 100 chart multiple times due to its popularity around Halloween. \"Thriller\" is a disco-funk song The introduction features sound effects such as a creaking door, thunder, feet walking on wooden planks, winds and howling wolves. This gave Jackson at least one top-20 hit across seven consecutive decades from 1969 on the Billboard Hot 100. \"Thriller\" was certified platinum by the Recording Industry Association of America on December 4, 1989, for sales of over one million physical units in the US As of August 2016, the song had sold 4,024,398 copies in the US. \"Thriller\" is certified Diamond by the Recording Industry Association of America. It appears on several of Jackson's greatest-hits albums and has been covered by numerous artists. The song has returned to the Billboard Hot 100 chart multiple times due to its popularity around Halloween. \"Thriller\" has returned to the Billboard Hot 100 chart multiple times due to its popularity around Halloween"} +{"query": "who sang the hit song \"thriller\"?", "domain": "music", "doc": ". \"Thriller\" is certified Diamond by the Recording Industry Association of America. It appears on several of Jackson's greatest-hits albums and has been covered by numerous artists. The song has returned to the Billboard Hot 100 chart multiple times due to its popularity around Halloween. \"Thriller\" has returned to the Billboard Hot 100 chart multiple times due to its popularity around Halloween. It re-entered the Billboard Hot 100 in October 2013 at number 42, number 31 in November 2018, and number 19 in November 2021, its highest placement since 1984. This gave Jackson at least one top-20 hit across seven consecutive decades from 1969 on the Billboard Hot 100."} diff --git a/AgentQnA/retrieval_tool/README.md b/AgentQnA/retrieval_tool/README.md new file mode 100644 index 0000000000..fc5a2da43c --- /dev/null +++ b/AgentQnA/retrieval_tool/README.md @@ -0,0 +1,37 @@ +# Retrieval tool for agent + +The retrieval tool in this example is an OPEA megaservice that is comprised of a query embedder, a document retriever and a document reranker. + +## Launch microservices + +``` +bash launch_retrieval_tool.sh +``` + +## Index data into vector database + +In this example, we use an example jsonl file to ingest example documents into the vector database. For more ways to ingest data and the type of documents supported by OPEA dataprep microservices, please refer to the documentation in the opea-project/GenAIComps repo. + +1. create a conda env +2. Run commands below + +``` +bash run_ingest_data.sh +``` + +## Validate services + +``` +export ip_address=$(hostname -I | awk '{print $1}') +curl http://${ip_address}:8889/v1/retrievaltool -X POST -H "Content-Type: application/json" -d '{ + "text": "Taylor Swift hometown" + }' +``` + +## Consume retrieval tool + +The endpoint for the retrieval tool is + +``` +http://${ip_address}:8889/v1/retrievaltool +``` diff --git a/AgentQnA/retrieval_tool/index_data.py b/AgentQnA/retrieval_tool/index_data.py new file mode 100644 index 0000000000..410a83cfb1 --- /dev/null +++ b/AgentQnA/retrieval_tool/index_data.py @@ -0,0 +1,77 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import json +import os + +import requests +import tqdm + + +def get_args(): + parser = argparse.ArgumentParser(description="Index data") + parser.add_argument("--host_ip", type=str, default="localhost", help="Host IP") + parser.add_argument("--port", type=int, default=6007, help="Port") + parser.add_argument("--filedir", type=str, default=None, help="file directory") + parser.add_argument("--filename", type=str, default=None, help="file name") + parser.add_argument("--chunk_size", type=int, default=10000, help="Chunk size") + parser.add_argument("--chunk_overlap", type=int, default=0, help="Chunk overlap") + args = parser.parse_args() + return args + + +def split_jsonl_into_txts(jsonl_file): + docs = [] + n = 0 + with open(jsonl_file, "r") as f: + for line in f: + data = json.loads(line) + docs.append(data["doc"]) + return docs + + +def write_docs_to_disk(docs, output_folder): + output_files = [] + for i, text in enumerate(docs): + output = os.path.join(output_folder, str(i) + ".txt") + output_files.append(output) + with open(output, "w") as f: + f.write(text) + return output_files + + +def delete_files(files): + for file in files: + os.remove(file) + + +def main(): + args = get_args() + print(args) + + host_ip = args.host_ip + port = args.port + proxies = {"http": ""} + url = "http://{host_ip}:{port}/v1/dataprep".format(host_ip=host_ip, port=port) + + # Split jsonl file into json files + files = split_jsonl_into_txts(os.path.join(args.filedir, args.filename)) + file_list = write_docs_to_disk(files, args.filedir) + + print(file_list) + + for file in tqdm.tqdm(file_list): + print("Indexing file: ", file) + files = [("files", (f, open(f, "rb"))) for f in [file]] + payload = {"chunk_size": args.chunk_size, "chunk_overlap": args.chunk_overlap} + resp = requests.request("POST", url=url, headers={}, files=files, data=payload, proxies=proxies) + print(resp.text) + + print("Removing temp files....") + delete_files(file_list) + print("ALL DONE!") + + +if __name__ == "__main__": + main() diff --git a/AgentQnA/retrieval_tool/launch_retrieval_tool.sh b/AgentQnA/retrieval_tool/launch_retrieval_tool.sh new file mode 100644 index 0000000000..1a142abf8e --- /dev/null +++ b/AgentQnA/retrieval_tool/launch_retrieval_tool.sh @@ -0,0 +1,25 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +host_ip=$(hostname -I | awk '{print $1}') +export HF_CACHE_DIR=${HF_CACHE_DIR} +export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export no_proxy=${no_proxy} +export http_proxy=${http_proxy} +export https_proxy=${https_proxy} +export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" +export RERANK_MODEL_ID="BAAI/bge-reranker-base" +export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006" +export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808" +export REDIS_URL="redis://${host_ip}:6379" +export INDEX_NAME="rag-redis" +export MEGA_SERVICE_HOST_IP=${host_ip} +export EMBEDDING_SERVICE_HOST_IP=${host_ip} +export RETRIEVER_SERVICE_HOST_IP=${host_ip} +export RERANK_SERVICE_HOST_IP=${host_ip} +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool" +export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep" +export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get_file" +export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete_file" + +docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml up -d diff --git a/AgentQnA/retrieval_tool/run_ingest_data.sh b/AgentQnA/retrieval_tool/run_ingest_data.sh new file mode 100644 index 0000000000..c82769c84f --- /dev/null +++ b/AgentQnA/retrieval_tool/run_ingest_data.sh @@ -0,0 +1,7 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FILEDIR=${WORKDIR}/GenAIExamples/AgentQnA/example_data/ +FILENAME=test_docs_music.jsonl + +python3 index_data.py --filedir ${FILEDIR} --filename ${FILENAME} --host_ip $host_ip diff --git a/AgentQnA/tests/1_build_images.sh b/AgentQnA/tests/1_build_images.sh new file mode 100644 index 0000000000..abac17682b --- /dev/null +++ b/AgentQnA/tests/1_build_images.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +WORKPATH=$(dirname "$PWD") +export WORKDIR=$WORKPATH/../../ +echo "WORKDIR=${WORKDIR}" +export ip_address=$(hostname -I | awk '{print $1}') + + +function get_genai_comps() { + if [ ! -d "GenAIComps" ] ; then + git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ + fi +} + + +function build_docker_images_for_retrieval_tool(){ + cd $WORKDIR/GenAIExamples/DocIndexRetriever/docker_image_build/ + # git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ + get_genai_comps + echo "Build all the images with --no-cache..." + service_list="doc-index-retriever dataprep-redis embedding-tei retriever-redis reranking-tei" + docker compose -f build.yaml build ${service_list} --no-cache + docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + + docker images && sleep 1s +} + +function build_agent_docker_image() { + cd $WORKDIR/GenAIExamples/AgentQnA/docker_image_build/ + get_genai_comps + echo "Build agent image with --no-cache..." + service_list="agent-endpoint" + docker compose -f build.yaml build ${service_list} --no-cache +} + +function main() { + echo "==================== Build docker images for retrieval tool ====================" + build_docker_images_for_retrieval_tool + echo "==================== Build docker images for retrieval tool completed ====================" + + echo "==================== Build agent docker image ====================" + build_agent_docker_image + echo "==================== Build agent docker image completed ====================" +} + +main diff --git a/AgentQnA/tests/2_start_retrieval_tool.sh b/AgentQnA/tests/2_start_retrieval_tool.sh new file mode 100644 index 0000000000..df080e40d8 --- /dev/null +++ b/AgentQnA/tests/2_start_retrieval_tool.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +WORKPATH=$(dirname "$PWD") +export WORKDIR=$WORKPATH/../../ +echo "WORKDIR=${WORKDIR}" +export ip_address=$(hostname -I | awk '{print $1}') + +export HF_CACHE_DIR=$WORKDIR/hf_cache +if [ ! -d "$HF_CACHE_DIR" ]; then + echo "Creating HF_CACHE directory" + mkdir -p "$HF_CACHE_DIR" +fi + +function start_retrieval_tool() { + echo "Starting Retrieval tool" + cd $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool/ + bash launch_retrieval_tool.sh +} + +echo "==================== Start retrieval tool ====================" +start_retrieval_tool +sleep 20 # needed for downloading the models +echo "==================== Retrieval tool started ====================" diff --git a/AgentQnA/tests/3_ingest_data_and_validate_retrieval.sh b/AgentQnA/tests/3_ingest_data_and_validate_retrieval.sh new file mode 100644 index 0000000000..8794f8188d --- /dev/null +++ b/AgentQnA/tests/3_ingest_data_and_validate_retrieval.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e + +WORKPATH=$(dirname "$PWD") +export WORKDIR=$WORKPATH/../../ +echo "WORKDIR=${WORKDIR}" +export ip_address=$(hostname -I | awk '{print $1}') +export host_ip=$ip_address +echo "ip_address=${ip_address}" + + +function validate() { + local CONTENT="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT" + echo 0 + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + echo 1 + fi +} + +function ingest_data_and_validate() { + echo "Ingesting data" + cd $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool/ + echo $PWD + local CONTENT=$(bash run_ingest_data.sh) + local EXIT_CODE=$(validate "$CONTENT" "Data preparation succeeded" "dataprep-redis-server") + echo "$EXIT_CODE" + local EXIT_CODE="${EXIT_CODE:0-1}" + echo "return value is $EXIT_CODE" + if [ "$EXIT_CODE" == "1" ]; then + docker logs dataprep-redis-server + return 1 + fi +} + +function validate_retrieval_tool() { + echo "----------------Test retrieval tool ----------------" + local CONTENT=$(http_proxy="" curl http://${ip_address}:8889/v1/retrievaltool -X POST -H "Content-Type: application/json" -d '{ + "text": "Who sang Thriller" + }') + local EXIT_CODE=$(validate "$CONTENT" "Thriller" "retrieval-tool") + + if [ "$EXIT_CODE" == "1" ]; then + docker logs retrievaltool-xeon-backend-server + exit 1 + fi +} + +function main(){ + + echo "==================== Ingest data ====================" + ingest_data_and_validate + echo "==================== Data ingestion completed ====================" + + echo "==================== Validate retrieval tool ====================" + validate_retrieval_tool + echo "==================== Retrieval tool validated ====================" +} + +main diff --git a/AgentQnA/tests/4_launch_and_validate_agent_openai.sh b/AgentQnA/tests/4_launch_and_validate_agent_openai.sh new file mode 100644 index 0000000000..fb9a8949f1 --- /dev/null +++ b/AgentQnA/tests/4_launch_and_validate_agent_openai.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +echo "OPENAI_API_KEY=${OPENAI_API_KEY}" + +WORKPATH=$(dirname "$PWD") +export WORKDIR=$WORKPATH/../../ +echo "WORKDIR=${WORKDIR}" +export ip_address=$(hostname -I | awk '{print $1}') +export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ + +function start_agent_and_api_server() { + echo "Starting CRAG server" + docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0 + + echo "Starting Agent services" + cd $WORKDIR/GenAIExamples/AgentQnA/docker/openai + bash launch_agent_service_openai.sh +} + +function validate() { + local CONTENT="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT" + echo 0 + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + echo 1 + fi +} + +function validate_agent_service() { + echo "----------------Test agent ----------------" + local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{ + "query": "Tell me about Michael Jackson song thriller" + }') + local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint") + docker logs react-agent-endpoint + if [ "$EXIT_CODE" == "1" ]; then + exit 1 + fi + +} + +function main() { + echo "==================== Start agent ====================" + start_agent_and_api_server + echo "==================== Agent started ====================" + + echo "==================== Validate agent service ====================" + validate_agent_service + echo "==================== Agent service validated ====================" +} + +main diff --git a/AgentQnA/tests/4_launch_and_validate_agent_tgi.sh b/AgentQnA/tests/4_launch_and_validate_agent_tgi.sh new file mode 100644 index 0000000000..03f20a9666 --- /dev/null +++ b/AgentQnA/tests/4_launch_and_validate_agent_tgi.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e + +WORKPATH=$(dirname "$PWD") +export WORKDIR=$WORKPATH/../../ +echo "WORKDIR=${WORKDIR}" +export ip_address=$(hostname -I | awk '{print $1}') +export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ +export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + +export HF_CACHE_DIR=$WORKDIR/hf_cache +if [ ! -d "$HF_CACHE_DIR" ]; then + mkdir -p "$HF_CACHE_DIR" +fi +ls $HF_CACHE_DIR + + +function start_agent_and_api_server() { + echo "Starting CRAG server" + docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0 + + echo "Starting Agent services" + cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi + bash launch_agent_service_tgi_gaudi.sh +} + +function validate() { + local CONTENT="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT" + echo 0 + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + echo 1 + fi +} + +function validate_agent_service() { + echo "----------------Test agent ----------------" + # local CONTENT=$(http_proxy="" curl http://${ip_address}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{ + # "query": "Tell me about Michael Jackson song thriller" + # }') + # local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint") + # docker logs docgrader-agent-endpoint + # if [ "$EXIT_CODE" == "1" ]; then + # exit 1 + # fi + + local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{ + "query": "Tell me about Michael Jackson song thriller" + }') + local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint") + docker logs react-agent-endpoint + if [ "$EXIT_CODE" == "1" ]; then + exit 1 + fi + +} + +function main() { + echo "==================== Start agent ====================" + start_agent_and_api_server + echo "==================== Agent started ====================" + + echo "==================== Validate agent service ====================" + validate_agent_service + echo "==================== Agent service validated ====================" +} + +main diff --git a/AgentQnA/tests/_test_compose_openai_on_xeon.sh b/AgentQnA/tests/_test_compose_openai_on_xeon.sh index ddb64c7061..a1c556499b 100644 --- a/AgentQnA/tests/_test_compose_openai_on_xeon.sh +++ b/AgentQnA/tests/_test_compose_openai_on_xeon.sh @@ -3,73 +3,48 @@ # SPDX-License-Identifier: Apache-2.0 set -e -echo "IMAGE_REPO=${IMAGE_REPO}" -echo "OPENAI_API_KEY=${OPENAI_API_KEY}" +echo "OPENAI_API_KEY=${OPENAI_API_KEY}" WORKPATH=$(dirname "$PWD") export WORKDIR=$WORKPATH/../../ echo "WORKDIR=${WORKDIR}" export ip_address=$(hostname -I | awk '{print $1}') export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ -function build_agent_docker_image() { - cd $WORKDIR - if [ ! -d "GenAIComps" ] ; then - git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ - fi - cd GenAIComps - echo PWD: $(pwd) - docker build -t opea/comps-agent-langchain:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/agent/langchain/Dockerfile . +function stop_agent_and_api_server() { + echo "Stopping CRAG server" + docker stop $(docker ps -q --filter ancestor=docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0) + echo "Stopping Agent services" + docker stop $(docker ps -q --filter ancestor=opea/comps-agent-langchain:latest) } -function start_services() { - echo "Starting CRAG server" - docker run -d -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0 - echo "Starting Agent services" - cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon/ - bash launch_agent_service_openai.sh +function stop_retrieval_tool() { + echo "Stopping Retrieval tool" + docker compose -f $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool/docker/docker-compose-retrieval-tool.yaml down } -function validate() { - local CONTENT="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" +echo "=================== #1 Building docker images====================" +bash 1_build_images.sh +echo "=================== #1 Building docker images completed====================" - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT" - echo 0 - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - echo 1 - fi -} +echo "=================== #2 Start retrieval tool====================" +bash 2_start_retrieval_tool.sh +echo "=================== #2 Retrieval tool started====================" +echo "=================== #3 Ingest data and validate retrieval====================" +bash 3_ingest_data_and_validate_retrieval.sh +echo "=================== #3 Data ingestion and validation completed====================" -function run_tests() { - echo "----------------Test supervisor agent ----------------" - local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{ - "query": "Most recent album by Taylor Swift" - }') - local EXIT_CODE=$(validate "$CONTENT" "Taylor" "react-agent-endpoint") - docker logs react-agent-endpoint - if [ "$EXIT_CODE" == "1" ]; then - exit 1 - fi +echo "=================== #4 Start agent and API server====================" +bash 4_launch_and_validate_agent_openai.sh +echo "=================== #4 Agent test passed ====================" -} +echo "=================== #5 Stop agent and API server====================" +stop_agent_and_api_server +echo "=================== #5 Agent and API server stopped====================" -function stop_services() { - echo "Stopping CRAG server" - docker stop $(docker ps -q --filter ancestor=docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0) - echo "Stopping Agent services" - docker stop $(docker ps -q --filter ancestor=opea/comps-agent-langchain:latest) -} - -function main() { - build_agent_docker_image - start_services - run_tests - stop_services -} +echo "=================== #6 Stop retrieval tool====================" +stop_retrieval_tool +echo "=================== #6 Retrieval tool stopped====================" -main +echo "ALL DONE!" diff --git a/AgentQnA/tests/test_compose_on_gaudi.sh b/AgentQnA/tests/test_compose_on_gaudi.sh new file mode 100644 index 0000000000..72d0fa0b1b --- /dev/null +++ b/AgentQnA/tests/test_compose_on_gaudi.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e + +WORKPATH=$(dirname "$PWD") +export WORKDIR=$WORKPATH/../../ +echo "WORKDIR=${WORKDIR}" +export ip_address=$(hostname -I | awk '{print $1}') +export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ + +function stop_crag() { + cid=$(docker ps -aq --filter "name=kdd-cup-24-crag-service") + echo "Stopping container kdd-cup-24-crag-service with cid $cid" + if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi +} + +function stop_agent_docker() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi/ + # docker compose -f compose.yaml down + container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + echo "Stopping container $container_name" + if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi + done +} + +function stop_retrieval_tool() { + echo "Stopping Retrieval tool" + local RETRIEVAL_TOOL_PATH=$WORKPATH/../DocIndexRetriever + cd $RETRIEVAL_TOOL_PATH/docker_compose/intel/cpu/xeon/ + # docker compose -f compose.yaml down + container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + echo "Stopping container $container_name" + if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi + done +} +echo "workpath: $WORKPATH" +echo "=================== Stop containers ====================" +stop_crag +stop_agent_docker +stop_retrieval_tool + +cd $WORKPATH/tests + +echo "=================== #1 Building docker images====================" +bash 1_build_images.sh +echo "=================== #1 Building docker images completed====================" + +echo "=================== #2 Start retrieval tool====================" +bash 2_start_retrieval_tool.sh +echo "=================== #2 Retrieval tool started====================" + +echo "=================== #3 Ingest data and validate retrieval====================" +bash 3_ingest_data_and_validate_retrieval.sh +echo "=================== #3 Data ingestion and validation completed====================" + +echo "=================== #4 Start agent and API server====================" +bash 4_launch_and_validate_agent_tgi.sh +echo "=================== #4 Agent test passed ====================" + +echo "=================== #5 Stop agent and API server====================" +stop_crag +stop_docker +stop_retrieval_tool +echo "=================== #5 Agent and API server stopped====================" + +echo "ALL DONE!" diff --git a/AgentQnA/tools/worker_agent_tools.py b/AgentQnA/tools/worker_agent_tools.py new file mode 100644 index 0000000000..1dfdb8409e --- /dev/null +++ b/AgentQnA/tools/worker_agent_tools.py @@ -0,0 +1,27 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os + +import requests + + +def search_knowledge_base(query: str) -> str: + """Search the knowledge base for a specific query.""" + url = os.environ.get("RETRIEVAL_TOOL_URL") + print(url) + proxies = {"http": ""} + payload = { + "text": query, + } + response = requests.post(url, json=payload, proxies=proxies) + print(response) + docs = response.json()["documents"] + context = "" + for i, doc in enumerate(docs): + if i == 0: + context = doc + else: + context += "\n" + doc + print(context) + return context diff --git a/AgentQnA/tools/worker_agent_tools.yaml b/AgentQnA/tools/worker_agent_tools.yaml index 905106ee35..32d19562cc 100644 --- a/AgentQnA/tools/worker_agent_tools.yaml +++ b/AgentQnA/tools/worker_agent_tools.yaml @@ -1,5 +1,11 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -duckduckgo_search: - callable_api: ddg-search +search_knowledge_base: + description: Search knowledge base for a given query. Returns text related to the query. + callable_api: worker_agent_tools.py:search_knowledge_base + args_schema: + query: + type: str + description: query + return_output: retrieved_data diff --git a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml index 71e894d559..5f66bb7423 100644 --- a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml @@ -27,7 +27,7 @@ services: INDEX_NAME: ${INDEX_NAME} TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT} tei-embedding-service: - image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 container_name: tei-embedding-server ports: - "6006:80" @@ -80,7 +80,7 @@ services: restart: unless-stopped tei-reranking-service: - image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 container_name: tei-reranking-server ports: - "8808:80" diff --git a/DocIndexRetriever/docker_image_build/build.yaml b/DocIndexRetriever/docker_image_build/build.yaml index 07005e89ae..3ed44fa24a 100644 --- a/DocIndexRetriever/docker_image_build/build.yaml +++ b/DocIndexRetriever/docker_image_build/build.yaml @@ -21,7 +21,7 @@ services: build: context: GenAIComps dockerfile: comps/retrievers/redis/langchain/Dockerfile - extends: chatqna + extends: doc-index-retriever image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} reranking-tei: build: @@ -29,9 +29,9 @@ services: dockerfile: comps/reranks/tei/Dockerfile extends: doc-index-retriever image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest} - dataprep-on-ray-redis: + dataprep-redis: build: context: GenAIComps - dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile + dockerfile: comps/dataprep/redis/langchain/Dockerfile extends: doc-index-retriever - image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}