notebooks/PromptLibCustomTask.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMn9OZRPVtMT3BRTm046gQQ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luyug/QuCo/blob/main/PromptLibCustomTask.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bJn91P8QbyDX",
        "outputId": "75183eb8-5193-4df7-9949-3b3b28927532"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'prompt-lib'...\n",
            "remote: Enumerating objects: 469, done.\u001b[K\n",
            "remote: Counting objects: 100% (469/469), done.\u001b[K\n",
            "remote: Compressing objects: 100% (265/265), done.\u001b[K\n",
            "remote: Total 469 (delta 277), reused 383 (delta 195), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (469/469), 2.02 MiB | 14.35 MiB/s, done.\n",
            "Resolving deltas: 100% (277/277), done.\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Processing ./prompt-lib\n",
            "\u001b[33m  DEPRECATION: A future pip version will change local packages to be built in-place without first copying to a temporary directory. We recommend you use --use-feature=in-tree-build to test your packages with this new behavior before it becomes the default.\n",
            "   pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.\u001b[0m\n",
            "Requirement already satisfied: openai==0.23.0 in /usr/local/lib/python3.7/dist-packages (from prompt-lib==0.1) (0.23.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from prompt-lib==0.1) (1.3.5)\n",
            "Requirement already satisfied: tqdm==4.49.0 in /usr/local/lib/python3.7/dist-packages (from prompt-lib==0.1) (4.49.0)\n",
            "Requirement already satisfied: wandb==0.13.2 in /usr/local/lib/python3.7/dist-packages (from prompt-lib==0.1) (0.13.2)\n",
            "Requirement already satisfied: pandas-stubs>=1.1.0.11 in /usr/local/lib/python3.7/dist-packages (from openai==0.23.0->prompt-lib==0.1) (1.2.0.62)\n",
            "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.7/dist-packages (from openai==0.23.0->prompt-lib==0.1) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from openai==0.23.0->prompt-lib==0.1) (4.1.1)\n",
            "Requirement already satisfied: openpyxl>=3.0.7 in /usr/local/lib/python3.7/dist-packages (from openai==0.23.0->prompt-lib==0.1) (3.0.10)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from openai==0.23.0->prompt-lib==0.1) (1.21.6)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (5.4.8)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (1.3.2)\n",
            "Requirement already satisfied: pathtools in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (0.1.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (57.4.0)\n",
            "Requirement already satisfied: GitPython>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (3.1.29)\n",
            "Requirement already satisfied: shortuuid>=0.5.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (1.0.11)\n",
            "Requirement already satisfied: protobuf<4.0dev,>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (3.19.6)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (2.3)\n",
            "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (1.9.0)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (6.0)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (0.4.0)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (1.15.0)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.2->prompt-lib==0.1) (7.1.2)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.7/dist-packages (from GitPython>=1.0.0->wandb==0.13.2->prompt-lib==0.1) (4.0.10)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb==0.13.2->prompt-lib==0.1) (5.0.0)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.7/dist-packages (from openpyxl>=3.0.7->openai==0.23.0->prompt-lib==0.1) (1.1.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->prompt-lib==0.1) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->prompt-lib==0.1) (2022.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->openai==0.23.0->prompt-lib==0.1) (2022.9.24)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->openai==0.23.0->prompt-lib==0.1) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->openai==0.23.0->prompt-lib==0.1) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->openai==0.23.0->prompt-lib==0.1) (1.24.3)\n",
            "Building wheels for collected packages: prompt-lib\n",
            "  Building wheel for prompt-lib (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for prompt-lib: filename=prompt_lib-0.1-py3-none-any.whl size=42476 sha256=d1c31f2a24b134adedb75f0d663f7d303914d5528dbf20625aeff58a66cc69d6\n",
            "  Stored in directory: /root/.cache/pip/wheels/24/68/12/bf7363e3bd22b539664450d1cffb4fd7d6ad2ac95c3166118e\n",
            "Successfully built prompt-lib\n",
            "Installing collected packages: prompt-lib\n",
            "  Attempting uninstall: prompt-lib\n",
            "    Found existing installation: prompt-lib 0.1\n",
            "    Uninstalling prompt-lib-0.1:\n",
            "      Successfully uninstalled prompt-lib-0.1\n",
            "Successfully installed prompt-lib-0.1\n"
          ]
        }
      ],
      "source": [
        "!rm -rf prompt-lib\n",
        "!git clone https://github.com/reasoning-machines/prompt-lib\n",
        "!pip install prompt-lib/"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "pDlHvAX5cX87"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Step 0: Export keys.**\n",
        "\n",
        "We need to have OPENAI_API_KEY in the environment because we are calling their model, so let's do that first."
      ],
      "metadata": {
        "id": "lyYrGET0cY7v"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ[\"OPENAI_API_KEY\"] = '' #@param {type:\"string\"}\n",
        "!export OPENAI_API_KEY"
      ],
      "metadata": {
        "id": "JpvriRnLb60p"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!echo $OPENAI_API_KEY"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7-6i8Kv7cEde",
        "outputId": "459d9e5f-4968-433a-9f4b-57508b5896ee"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import wandb\n",
        "wandb.init(mode=\"disabled\")\n",
        "# NOTE: prompt-lib has rich wandb logging. We are disabling it just for this notebook"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "lrImwxA_cGwP",
        "outputId": "ae80f3a4-9cc5-48df-f64b-f72dc4a68c23"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:wandb.jupyter:Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "r7gg0gFYcsHi"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Step 1: Task Configuration** \n",
        "\n",
        "\n",
        "`prompt-lib` provides a large number of tasks out of the box, and adding new tasks is trivial. In this notebook, we use a pre-registered task called `gsm_stream`. Here, `gsm` is the name of the task (GSM8k), and `stream` is just a tag that specifies that it's a chain-of-thought task. I saw the `stream` label in Wei et al. repo and have been using it for CoT like tasks for consistency. When we register a new task, we are free to give it any label with some restrictions (see README for details)."
      ],
      "metadata": {
        "id": "prbSoOalcssD"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "17KIDnnxeH-W"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "task_config_dict = {\n",
        "    \"task_id\": \"gsm_stream\",\n",
        "    \"tag\": \"gsm_stream\",\n",
        "    \"num_prompt_examples\": 5,\n",
        "    \"max_tokens\": 600,\n",
        "    \"num_questions_per_thread\": 100,\n",
        "    \"seed\": 0,\n",
        "    \"is_cot_task\": True,\n",
        "    \"model_name\": \"code-davinci-002\",\n",
        "    \"max_requests_per_min\": 20,\n",
        "    \"cached_timestamp\": None,\n",
        "    \"temperature\": 0.0,\n",
        "    \"is_debug\": True,\n",
        "    \"prompt_config\": {\n",
        "        \"question_prefix\": \"Q: \",\n",
        "        \"answer_prefix\": \"A: \",\n",
        "        \"final_answer_prefix\": \"The answer is \",\n",
        "        \"intra_example_sep\": \"\\n\",\n",
        "        \"inter_example_sep\": \"\\n\\n\"\n",
        "    },\n",
        "    \"eval_function\": \"get_exact_match_acc\"\n",
        "}"
      ],
      "metadata": {
        "id": "6BIlmwdxct6q"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "RuBDP1-Dc3pG"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's look at some of the important fields fields in detail:\n",
        "\n",
        "\n",
        "\n",
        "*   `task_id` identifies the task. Crucially, this assumes that you have a file called `data/tasks/gsm.jsonl` with two fields: `input` and `target`. The task file is always extracted from the task name by doing `task_name.split('_')[0]`. \n",
        "\n",
        "Also, this assumes that you have `gsm_stream` task registered. See the README for details on how a task is registered. Briefly, task registration requires specifying a name for the task, and the prompt it'll use. \n",
        "*   `prompt_config` outlines how the prompt is made. \n",
        "\n",
        "\n",
        "Let's add a link to the dataset we need for inference, `gsm.jsonl`. gsm.jsonl is already included in prompt-lib, but not in the current directory. We add it as a soft-link.\n"
      ],
      "metadata": {
        "id": "1e9gODr0c6df"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!mkdir -p data/tasks\n",
        "%cd data/tasks\n",
        "!ln -s ../../prompt-lib/data/tasks/gsm.jsonl gsm.jsonl\n",
        "%cd ../.."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Ojg7u4RdTFV",
        "outputId": "9dca8c3a-459e-4fd4-b9ad-51584d4b5520"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/data/tasks\n",
            "ln: failed to create symbolic link 'gsm.jsonl': File exists\n",
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!ls data/tasks\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "62ZNThwZdsRw",
        "outputId": "e3b3b83e-6dd6-4d2a-ad4c-b105516236d7"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "gsm.jsonl\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Step 2: Running the job**"
      ],
      "metadata": {
        "id": "VvWaKcfHe_Rt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from prompt_lib.prompts.utils import TaskConfig\n",
        "from prompt_lib.inference import inference_loop\n",
        "task_config = TaskConfig.from_config_dict(task_config_dict)\n"
      ],
      "metadata": {
        "id": "SfrTjcztdZOt"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "outputs = inference_loop(task_config, num_inference_examples=3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hjMKPMPjfn8S",
        "outputId": "3a001edc-398e-40ac-cca3-e7c6c6c2e8c9"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  0%|          | 0/1 [00:00<?, ?it/s]\n",
            "Querying openai [thread_id=0]:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
            "Querying openai [thread_id=0]:  33%|███▎      | 1/3 [00:05<00:11,  5.79s/it]\u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Janet’s ducks lay 16 eggs per day. She eats 3 for breakfast and bakes 4 for muffins. So she has 16 - 3 - 4 = 9 eggs left. She sells them for $2 each. So she makes 9 * 2 = 18 dollars. The answer is 18\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Querying openai [thread_id=0]:  67%|██████▋   | 2/3 [00:08<00:04,  4.86s/it]\u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A robe takes 2 bolts of blue fiber and half that much white fiber.  Half of 2 is 1.  So it takes 2 + 1 = 3 bolts in total.  The answer is 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Querying openai [thread_id=0]: 100%|██████████| 3/3 [00:12<00:00,  4.13s/it]\n",
            "100%|██████████| 1/1 [00:12<00:00, 12.42s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Josh bought the house for 80,000 dollars. He put in 50,000 dollars in repairs. The house increased in value by 150%. So the house is now worth 80,000 + 50,000 + (50,000 * 150%) = 80,000 + 50,000 + 75,000 = 205,000 dollars. He made a profit of 205,000 - 80,000 - 50,000 = 75,000 dollars. The answer is 75,000\n",
            "Accuracy: 66.67\n",
            "Accuracy: 66.67\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "outputs.head(3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "lQ8j0xPDfsuU",
        "outputId": "73e0a261-e550-4a4e-f530-194a399b6ef9"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              prompt  \\\n",
              "0  Q: Olivia has $23. She bought five bagels for ...   \n",
              "1  Q: Olivia has $23. She bought five bagels for ...   \n",
              "2  Q: Olivia has $23. She bought five bagels for ...   \n",
              "\n",
              "                                            question  answer  \\\n",
              "0  \\nQ: Janet’s ducks lay 16 eggs per day. She ea...      18   \n",
              "1  \\nQ: A robe takes 2 bolts of blue fiber and ha...       3   \n",
              "2  \\nQ: Josh decides to try flipping a house.  He...   70000   \n",
              "\n",
              "                                    generated_answer  \\\n",
              "0  Janet’s ducks lay 16 eggs per day. She eats 3 ...   \n",
              "1  A robe takes 2 bolts of blue fiber and half th...   \n",
              "2  Josh bought the house for 80,000 dollars. He p...   \n",
              "\n",
              "                                     entire_response  is_correct  \n",
              "0  Janet’s ducks lay 16 eggs per day. She eats 3 ...           1  \n",
              "1  A robe takes 2 bolts of blue fiber and half th...           1  \n",
              "2  Josh bought the house for 80,000 dollars. He p...           0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4e26240e-3cab-4276-8948-e8a6ff62264b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>prompt</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>generated_answer</th>\n",
              "      <th>entire_response</th>\n",
              "      <th>is_correct</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Q: Olivia has $23. She bought five bagels for ...</td>\n",
              "      <td>\\nQ: Janet’s ducks lay 16 eggs per day. She ea...</td>\n",
              "      <td>18</td>\n",
              "      <td>Janet’s ducks lay 16 eggs per day. She eats 3 ...</td>\n",
              "      <td>Janet’s ducks lay 16 eggs per day. She eats 3 ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Q: Olivia has $23. She bought five bagels for ...</td>\n",
              "      <td>\\nQ: A robe takes 2 bolts of blue fiber and ha...</td>\n",
              "      <td>3</td>\n",
              "      <td>A robe takes 2 bolts of blue fiber and half th...</td>\n",
              "      <td>A robe takes 2 bolts of blue fiber and half th...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Q: Olivia has $23. She bought five bagels for ...</td>\n",
              "      <td>\\nQ: Josh decides to try flipping a house.  He...</td>\n",
              "      <td>70000</td>\n",
              "      <td>Josh bought the house for 80,000 dollars. He p...</td>\n",
              "      <td>Josh bought the house for 80,000 dollars. He p...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4e26240e-3cab-4276-8948-e8a6ff62264b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"https://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4e26240e-3cab-4276-8948-e8a6ff62264b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4e26240e-3cab-4276-8948-e8a6ff62264b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "BsKugKvBe7tX"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**That's it! We can use the output for further analysis. A copy of the outputs is also saved at `data/logs/` with necessary hyperparameters in the path**"
      ],
      "metadata": {
        "id": "X63vQ01ifzOZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!ls -R data/logs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HuvtuidBf1M3",
        "outputId": "22ddee2b-2ca7-4e06-ffd6-7e548b5172b6"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "data/logs:\n",
            "gsm_stream\n",
            "\n",
            "data/logs/gsm_stream:\n",
            "code-davinci-002\n",
            "\n",
            "data/logs/gsm_stream/code-davinci-002:\n",
            "temp_0.0\n",
            "\n",
            "data/logs/gsm_stream/code-davinci-002/temp_0.0:\n",
            "seed_0\n",
            "\n",
            "data/logs/gsm_stream/code-davinci-002/temp_0.0/seed_0:\n",
            "k_5\n",
            "\n",
            "data/logs/gsm_stream/code-davinci-002/temp_0.0/seed_0/k_5:\n",
            "gsm_stream\n",
            "\n",
            "data/logs/gsm_stream/code-davinci-002/temp_0.0/seed_0/k_5/gsm_stream:\n",
            "2022-11-30_07-44-30\n",
            "\n",
            "data/logs/gsm_stream/code-davinci-002/temp_0.0/seed_0/k_5/gsm_stream/2022-11-30_07-44-30:\n",
            "outputs.jsonl  outputs_part0.jsonl  task_config.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "bnKYqqe0f_jM"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}