dataset_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0d684f71-ae38-4b4e-865c-731f19bb1699",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "83c4972d-fada-4f96-8f0b-6b22d3daaa91",
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_token = \"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\"\n",
    "hf_folder = HfFolder()\n",
    "hf_folder.save_token(hf_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "354e24bd-54e0-4e29-b256-55d476237f98",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset('teknium/OpenHermes-2.5', use_auth_token=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6535d0b7-4462-47da-98e7-65c9c4247038",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(dataset['train'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eb704ac-0f30-4adf-b2cc-4ee1b5718dac",
   "metadata": {},
   "outputs": [],
   "source": [
    "subset_df = df.sample(n=15000, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af87058e-c2ad-470d-a1c0-b180d484dbe3",
   "metadata": {},
   "outputs": [],
   "source": [
    "subset_df.to_json('OpenHermes_subset_15k_table.json', orient='table')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "040e2759-d265-4f94-adea-c480521afb26",
   "metadata": {},
   "outputs": [],
   "source": [
    "fp=open('OpenHermes_subset_15k_table.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19ee2cc7-3c6b-4a35-92af-0cd6d4eec42c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dc2ae78-54eb-4893-b449-1279167d59f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=json.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b633ffce-48bd-4631-bb5d-3a0cca94fadf",
   "metadata": {},
   "outputs": [],
   "source": [
    "data1=data['data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e653092-907c-4443-a8d8-719cfd3ac402",
   "metadata": {},
   "outputs": [],
   "source": [
    "global_list=[]\n",
    "for x in data1:\n",
    "    temp_list=x['conversations']\n",
    "    temp_list1=[]\n",
    "    for index,y in enumerate(temp_list):\n",
    "        if y['from']!='system' and index==0:\n",
    "            temp_dict={}\n",
    "            temp_dict[\"role\"]=\"system\"\n",
    "            temp_dict[\"value\"]=\"You are a helpful AI assitant\"\n",
    "            temp_list1.append(temp_dict)\n",
    "        if y['from']=='system':\n",
    "            temp_dict={}\n",
    "            temp_dict[\"role\"]=y['from']\n",
    "            temp_dict[\"value\"]=y['value']\n",
    "            temp_list1.append(temp_dict)\n",
    "            continue\n",
    "        temp_dict={}\n",
    "        temp_dict[\"role\"]=y['from']\n",
    "        temp_dict[\"value\"]=y['value']\n",
    "        temp_list1.append(temp_dict) \n",
    "    global_list.append({\"conversations\":temp_list1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3344f84d-d913-4688-bf11-968bbe92ac05",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_file_path = \"final_openhermes_15k.jsonl\"\n",
    "with open(output_file_path, \"w\") as outfile:\n",
    "    for data_dict in global_list:\n",
    "        json.dump(data_dict, outfile)\n",
    "        outfile.write('\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}