baseline-deepseek.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zQAstXRA5Elp",
        "outputId": "a32845cb-7ab1-44af-8029-2f5f76820b10"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'small-coder'...\n",
            "remote: Enumerating objects: 78, done.\u001b[K\n",
            "remote: Counting objects: 100% (78/78), done.\u001b[K\n",
            "remote: Compressing objects: 100% (57/57), done.\u001b[K\n",
            "remote: Total 78 (delta 35), reused 51 (delta 16), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (78/78), 481.33 KiB | 4.81 MiB/s, done.\n",
            "Resolving deltas: 100% (35/35), done.\n",
            "/content/small-coder\n"
          ]
        }
      ],
      "source": [
        "![ ! -f requirements.txt ] && git clone https://github.com/anudaweerasinghe/small-coder.git\n",
        "%cd /content/small-coder"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "gEAAaukh5Elr"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "\n",
        "! pip3 install -r requirements.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "biOzMPt4y5IY",
        "outputId": "affa8cf2-d207-43c3-b0e8-8a03d24ce41f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'bigcode-evaluation-harness'...\n",
            "remote: Enumerating objects: 4113, done.\u001b[K\n",
            "remote: Counting objects: 100% (1590/1590), done.\u001b[K\n",
            "remote: Compressing objects: 100% (484/484), done.\u001b[K\n",
            "remote: Total 4113 (delta 1259), reused 1283 (delta 1097), pack-reused 2523\u001b[K\n",
            "Receiving objects: 100% (4113/4113), 817.80 KiB | 6.10 MiB/s, done.\n",
            "Resolving deltas: 100% (2798/2798), done.\n",
            "/content/small-coder/bigcode-evaluation-harness\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/bigcode-project/bigcode-evaluation-harness.git\n",
        "%cd bigcode-evaluation-harness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "NrdHKbyszgyy"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "\n",
        "!pip install -e ."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tgZuU9OGzmhb",
        "outputId": "154e7e13-375f-4546-8302-3317532744ae"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accelerate configuration saved at /root/.cache/huggingface/accelerate/default_config.yaml\n"
          ]
        }
      ],
      "source": [
        "!accelerate config default\n",
        "!mv /root/.cache/huggingface/accelerate/default_config.yaml /content/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oRRgCeYp0rrR",
        "outputId": "e92a2e5d-6978-4a4f-e111-73391a9875b8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The following values were not passed to `accelerate launch` and had defaults used instead:\n",
            "\t`--num_processes` was set to a value of `1`\n",
            "\t`--num_machines` was set to a value of `1`\n",
            "\t`--mixed_precision` was set to a value of `'no'`\n",
            "\t`--dynamo_backend` was set to a value of `'no'`\n",
            "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n",
            "2024-04-13 00:21:45.402544: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-04-13 00:21:45.402590: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-04-13 00:21:45.403895: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-04-13 00:21:46.523923: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Selected Tasks: ['humaneval']\n",
            "Loading model in fp32\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py:466: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py:720: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n",
            "  warnings.warn(\n",
            "number of problems for this task is 164\n",
            "100% 164/164 [10:57<00:00,  4.01s/it]\n",
            "Evaluating generations...\n",
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "{\n",
            "  \"humaneval\": {\n",
            "    \"pass@1\": 0.5975609756097561\n",
            "  },\n",
            "  \"config\": {\n",
            "    \"prefix\": \"\",\n",
            "    \"do_sample\": true,\n",
            "    \"temperature\": 0.2,\n",
            "    \"top_k\": 0,\n",
            "    \"top_p\": 0.95,\n",
            "    \"n_samples\": 1,\n",
            "    \"eos\": \"<|endoftext|>\",\n",
            "    \"seed\": 0,\n",
            "    \"model\": \"deepseek-ai/deepseek-coder-1.3b-instruct\",\n",
            "    \"modeltype\": \"causal\",\n",
            "    \"peft_model\": null,\n",
            "    \"revision\": null,\n",
            "    \"use_auth_token\": false,\n",
            "    \"trust_remote_code\": false,\n",
            "    \"tasks\": \"humaneval\",\n",
            "    \"instruction_tokens\": null,\n",
            "    \"batch_size\": 1,\n",
            "    \"max_length_generation\": 1024,\n",
            "    \"precision\": \"fp32\",\n",
            "    \"load_in_8bit\": false,\n",
            "    \"load_in_4bit\": false,\n",
            "    \"left_padding\": false,\n",
            "    \"limit\": null,\n",
            "    \"limit_start\": 0,\n",
            "    \"save_every_k_tasks\": -1,\n",
            "    \"postprocess\": true,\n",
            "    \"allow_code_execution\": true,\n",
            "    \"generation_only\": false,\n",
            "    \"load_generations_path\": null,\n",
            "    \"load_data_path\": null,\n",
            "    \"metric_output_path\": \"evaluation_results.json\",\n",
            "    \"save_generations\": false,\n",
            "    \"load_generations_intermediate_paths\": null,\n",
            "    \"save_generations_path\": \"generations.json\",\n",
            "    \"save_references\": false,\n",
            "    \"save_references_path\": \"references.json\",\n",
            "    \"prompt\": \"prompt\",\n",
            "    \"max_memory_per_gpu\": null,\n",
            "    \"check_references\": false\n",
            "  }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "!accelerate launch  main.py \\\n",
        "  --model \"deepseek-ai/deepseek-coder-1.3b-instruct\" \\\n",
        "  --max_length_generation 1024 \\\n",
        "  --tasks humaneval \\\n",
        "  --temperature 0.2 \\\n",
        "  --n_samples 1 \\\n",
        "  --batch_size 1 \\\n",
        "  --allow_code_execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bq6rDO_B4Lrg",
        "outputId": "881b57be-45af-40fd-eb93-853736565aa2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The following values were not passed to `accelerate launch` and had defaults used instead:\n",
            "\t`--num_processes` was set to a value of `1`\n",
            "\t`--num_machines` was set to a value of `1`\n",
            "\t`--mixed_precision` was set to a value of `'no'`\n",
            "\t`--dynamo_backend` was set to a value of `'no'`\n",
            "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n",
            "2024-04-13 00:35:32.025759: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-04-13 00:35:32.025810: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-04-13 00:35:32.027219: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-04-13 00:35:33.105846: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Selected Tasks: ['mbpp']\n",
            "Loading model in fp32\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py:466: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py:720: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n",
            "  warnings.warn(\n",
            "Downloading readme: 100% 9.06k/9.06k [00:00<00:00, 21.6MB/s]\n",
            "Downloading data: 100% 87.2k/87.2k [00:01<00:00, 47.4kB/s]\n",
            "Downloading data: 100% 116k/116k [00:01<00:00, 63.2kB/s]\n",
            "Downloading data: 100% 25.1k/25.1k [00:00<00:00, 57.3kB/s]\n",
            "Downloading data: 100% 7.88k/7.88k [00:01<00:00, 6.18kB/s]\n",
            "Generating train split: 100% 374/374 [00:00<00:00, 11328.51 examples/s]\n",
            "Generating test split: 100% 500/500 [00:00<00:00, 125270.41 examples/s]\n",
            "Generating validation split: 100% 90/90 [00:00<00:00, 31530.85 examples/s]\n",
            "Generating prompt split: 100% 10/10 [00:00<00:00, 4167.63 examples/s]\n",
            "number of problems for this task is 500\n",
            " 30% 148/500 [08:18<25:12,  4.30s/it]"
          ]
        }
      ],
      "source": [
        "!accelerate launch  main.py \\\n",
        "  --model \"deepseek-ai/deepseek-coder-1.3b-instruct\" \\\n",
        "  --max_length_generation 256000 \\\n",
        "  --tasks mbpp \\\n",
        "  --temperature 0.2 \\\n",
        "  --n_samples 1 \\\n",
        "  --batch_size 1 \\\n",
        "  --allow_code_execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "WseLbEH04VhG",
        "outputId": "9f0aae80-66a8-47dd-cde2-11597e9f5d5d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The following values were not passed to `accelerate launch` and had defaults used instead:\n",
            "\t`--num_processes` was set to a value of `1`\n",
            "\t`--num_machines` was set to a value of `1`\n",
            "\t`--mixed_precision` was set to a value of `'no'`\n",
            "\t`--dynamo_backend` was set to a value of `'no'`\n",
            "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n",
            "2024-04-17 05:34:12.633007: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-04-17 05:34:12.633055: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-04-17 05:34:12.634409: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-04-17 05:34:13.649608: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Selected Tasks: ['humaneval']\n",
            "Loading model in fp32\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py:466: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n",
            "  warnings.warn(\n",
            "config.json: 100% 608/608 [00:00<00:00, 3.86MB/s]\n",
            "model.safetensors: 100% 2.20G/2.20G [00:17<00:00, 126MB/s]\n",
            "generation_config.json: 100% 124/124 [00:00<00:00, 789kB/s]\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py:720: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n",
            "  warnings.warn(\n",
            "tokenizer_config.json: 100% 1.29k/1.29k [00:00<00:00, 8.29MB/s]\n",
            "tokenizer.model: 100% 500k/500k [00:00<00:00, 169MB/s]\n",
            "tokenizer.json: 100% 1.84M/1.84M [00:00<00:00, 25.2MB/s]\n",
            "special_tokens_map.json: 100% 551/551 [00:00<00:00, 3.20MB/s]\n",
            "Downloading readme: 100% 6.52k/6.52k [00:00<00:00, 25.2MB/s]\n",
            "Downloading data: 100% 83.9k/83.9k [00:00<00:00, 483kB/s]\n",
            "Generating test split: 100% 164/164 [00:00<00:00, 2120.60 examples/s]\n",
            "number of problems for this task is 164\n",
            "100% 164/164 [10:55<00:00,  4.00s/it]\n",
            "Evaluating generations...\n",
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "{\n",
            "  \"humaneval\": {\n",
            "    \"pass@1\": 0.11585365853658537\n",
            "  },\n",
            "  \"config\": {\n",
            "    \"prefix\": \"\",\n",
            "    \"do_sample\": true,\n",
            "    \"temperature\": 0.2,\n",
            "    \"top_k\": 0,\n",
            "    \"top_p\": 0.95,\n",
            "    \"n_samples\": 1,\n",
            "    \"eos\": \"<|endoftext|>\",\n",
            "    \"seed\": 0,\n",
            "    \"model\": \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\",\n",
            "    \"modeltype\": \"causal\",\n",
            "    \"peft_model\": null,\n",
            "    \"revision\": null,\n",
            "    \"use_auth_token\": false,\n",
            "    \"trust_remote_code\": false,\n",
            "    \"tasks\": \"humaneval\",\n",
            "    \"instruction_tokens\": null,\n",
            "    \"batch_size\": 1,\n",
            "    \"max_length_generation\": 1024,\n",
            "    \"precision\": \"fp32\",\n",
            "    \"load_in_8bit\": false,\n",
            "    \"load_in_4bit\": false,\n",
            "    \"left_padding\": false,\n",
            "    \"limit\": null,\n",
            "    \"limit_start\": 0,\n",
            "    \"save_every_k_tasks\": -1,\n",
            "    \"postprocess\": true,\n",
            "    \"allow_code_execution\": true,\n",
            "    \"generation_only\": false,\n",
            "    \"load_generations_path\": null,\n",
            "    \"load_data_path\": null,\n",
            "    \"metric_output_path\": \"evaluation_results.json\",\n",
            "    \"save_generations\": false,\n",
            "    \"load_generations_intermediate_paths\": null,\n",
            "    \"save_generations_path\": \"generations.json\",\n",
            "    \"save_references\": false,\n",
            "    \"save_references_path\": \"references.json\",\n",
            "    \"prompt\": \"prompt\",\n",
            "    \"max_memory_per_gpu\": null,\n",
            "    \"check_references\": false\n",
            "  }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "# TinyLLama base model results: HumanEval\n",
        "!accelerate launch  main.py \\\n",
        "  --model \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\" \\\n",
        "  --max_length_generation 1024 \\\n",
        "  --tasks humaneval \\\n",
        "  --temperature 0.2 \\\n",
        "  --n_samples 1 \\\n",
        "  --batch_size 1 \\\n",
        "  --allow_code_execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7fbYDlS4OmRD"
      },
      "outputs": [],
      "source": [
        "# TinyLLama base model results: MBPP\n",
        "\n",
        "!accelerate launch  main.py \\\n",
        "  --model \"TinyLlama/TinyLlama-1.1B-python-v0.1\" \\\n",
        "  --max_length_generation 256000 \\\n",
        "  --tasks mbpp \\\n",
        "  --temperature 0.2 \\\n",
        "  --n_samples 1 \\\n",
        "  --batch_size 1 \\\n",
        "  --allow_code_execution"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}