run_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Knowledge Classification Result Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as mticker\n",
    "# from matplotlib import cm\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sources.utils import *\n",
    "from sources.process import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load model results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_name_list = [\"Llama2_7B\", \"Llama3_8B\", \"Llama3.1_8B\", \"Mistral7B\", \"Phi3.5_Mini\", \"SOLAR_10.7B\", \"Gemma_7B\", \"Gemma2_9B\"]\n",
    "model_name_list = [\"Llama3.1_8B\", \"Mistral7B\", \"Phi3.5_Mini\", \"SOLAR_10.7B\", \"Gemma2_9B\", 'gpt-4o-mini']\n",
    "# model_name_list = ['gpt-4o-mini']\n",
    "\n",
    "official_model_name = {\n",
    "    \"Llama3.1_8B\": \"Llama-3.1-8B-Instruct\", \n",
    "    \"Mistral7B\": \"Mistral-7B-Instruct-v0.3\",\n",
    "    \"Phi3.5_Mini\": \"Phi-3.5-mini-instruct\",\n",
    "    \"SOLAR_10.7B\": \"SOLAR-10.7B-Instruct-v1.0\",\n",
    "    \"Gemma2_9B\": \"gemma-2-9b-it\",\n",
    "    \"gpt-4o-mini\": \"GPT-4o mini\",\n",
    "}\n",
    "\n",
    "official_model_color = {\n",
    "    \"Llama3.1_8B\": \"blue\",\n",
    "    \"Mistral7B\": \"orange\",\n",
    "    \"Phi3.5_Mini\": \"red\",\n",
    "    \"SOLAR_10.7B\": \"purple\",\n",
    "    \"Gemma2_9B\": \"green\",\n",
    "    \"gpt-4o-mini\": \"black\",\n",
    "}\n",
    "\n",
    "\n",
    "official_model_checkpoint = {\n",
    "    \"Llama3.1_8B\": 2023, # https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct\n",
    "    \"Mistral7B\": 2024, # https://aimlapi.com/models/mistral-7b-instruct-v0-3 (unofficial)\n",
    "    \"Phi3.5_Mini\": 2023, # https://huggingface.co/microsoft/Phi-3.5-mini-instruct\n",
    "    \"SOLAR_10.7B\": \"purple\", \n",
    "    \"Gemma2_9B\": \"green\", \n",
    "    \"gpt-4o-mini\": 2023, # https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results_list(model_name_list, domain, mode):\n",
    "    results_list_temp_state = []\n",
    "\n",
    "    for t_state in [\"Dynamic\", \"Static\"]:\n",
    "        results_list = []\n",
    "\n",
    "        for model_name in tqdm(model_name_list):\n",
    "            bench, temp0_parsed_time, temp7_parsed_time = load_result(\n",
    "                                                                    model_name=model_name,\n",
    "                                                                    domain=domain,\n",
    "                                                                    temp_state=t_state,   \n",
    "                                                                    mode=mode\n",
    "                                                                    )\n",
    "\n",
    "            results, object_classification = classify_results_time(bench, temp0_parsed_time, temp7_parsed_time, model_name, domain, mode)\n",
    "            fine_grained_results, classification_indices = classify_results_time_fine_graining(bench, temp0_parsed_time, temp7_parsed_time, model_name, domain, mode)\n",
    "            \n",
    "            results_list.append(results)\n",
    "\n",
    "            sampling_results(fine_grained_results, classification_indices, model_name, domain, t_state)\n",
    "\n",
    "        results_list_temp_state.append(results_list)\n",
    "\n",
    "    return results_list_temp_state\n",
    "\n",
    "def get_results_list_invariant(model_name_list, domain, mode):\n",
    "\n",
    "    results_list = []\n",
    "\n",
    "    t_state = \"\"\n",
    "\n",
    "    for model_name in tqdm(model_name_list):\n",
    "        bench, temp0_parsed_time, temp7_parsed_time = load_result(\n",
    "                                                                model_name=model_name,\n",
    "                                                                domain=domain,\n",
    "                                                                temp_state=t_state, \n",
    "                                                                mode=mode, \n",
    "                                                                    )\n",
    "\n",
    "        \n",
    "        results, object_classification = classify_results_time(bench, temp0_parsed_time, temp7_parsed_time, model_name, domain, mode)\n",
    "        results_list.append(results)\n",
    "\n",
    "    return results_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making Plot with Time Variant Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# domains: General, Biomedical, Legal, CommonSense, Math\n",
    "domain = \"General\"\n",
    "\n",
    "mode ='generation'\n",
    "\n",
    "results_list_generation = get_results_list(model_name_list=model_name_list,\n",
    "                 domain=domain,\n",
    "                 mode=mode)\n",
    "\n",
    "mode ='QA'\n",
    "\n",
    "results_list_mcqa = get_results_list(model_name_list=model_name_list,\n",
    "                 domain=domain,\n",
    "                 mode=mode)\n",
    "\n",
    "mode ='TF'\n",
    "\n",
    "results_list_tf = get_results_list(model_name_list=model_name_list,\n",
    "                 domain=domain,\n",
    "                 mode=mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_histogram_time_model_total(model_name_list, results_list_change, bench_name, mode, changed_title=True):\n",
    "\n",
    "    years = sorted(results_list_change[0][0].keys())\n",
    "    # extract the year part\n",
    "    year_labels = [int(year.split('_')[-1]) for year in years]\n",
    "\n",
    "    categories = ['correct', 'partial_correct1', 'partial_correct2', 'incorrect']\n",
    "    colors = ['#3C6495', '#7E9BC7', '#B5C3DD', '#FDE58E']\n",
    "\n",
    "    # Create the plot\n",
    "    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 4.5))  # 두 개의 그래프를 한 줄에 배치\n",
    "\n",
    "    for ax, results_list in zip([ax1, ax2], results_list_change):\n",
    "\n",
    "        max_val = 0\n",
    "        min_val = 100\n",
    "        for model_name, results in zip(model_name_list, results_list):\n",
    "\n",
    "            data = []\n",
    "            for year in years:\n",
    "                counts = [results[year][category] for category in categories]\n",
    "                total = sum(counts)\n",
    "\n",
    "                knowns = 0\n",
    "                for category in categories:\n",
    "                    if \"correct\" in category:\n",
    "                        knowns += results[year][category]\n",
    "\n",
    "                knowns_percentage = knowns / total * 100\n",
    "                data.append(knowns_percentage)\n",
    "\n",
    "                if knowns_percentage > max_val:\n",
    "                    max_val = knowns_percentage\n",
    "                if knowns_percentage < min_val:\n",
    "                    min_val = knowns_percentage\n",
    "\n",
    "            # pointplot\n",
    "            ax.plot(year_labels, data, linestyle='-', linewidth=5,\n",
    "                    label=official_model_name[model_name] if ax == ax2 else \"\",\n",
    "                    color=official_model_color[model_name])\n",
    "\n",
    "        min_space = 35\n",
    "        if max_val - min_val <= min_space:\n",
    "            stride = min_space - (max_val-min_val)\n",
    "            min_val = int(min_val - stride / 2)\n",
    "            max_val = int(max_val + stride / 2)\n",
    "\n",
    "        min_val = 3 if min_val < 3 else min_val\n",
    "\n",
    "        ax.set_xticks(year_labels)\n",
    "        ax.set_xticklabels(year_labels, fontsize=15)\n",
    "\n",
    "        ax.set_ylim(min_val - 3, max_val + 3)\n",
    "        ax.set_yticklabels(ax.get_yticks(), fontsize=15)\n",
    "        ax.yaxis.set_major_formatter(mticker.FuncFormatter(lambda x, _: f'{int(x)}'))\n",
    "        ax.grid(True)\n",
    "        ax.margins(x=0.03)\n",
    "        ax.grid(False, axis='x')\n",
    "        ax.patch.set_facecolor('#f1f1f0')\n",
    "\n",
    "    if not changed_title:\n",
    "        # legend_properties = {'weight':'bold', 'size': 16}\n",
    "        legend_properties = {'size': 16}\n",
    "        fig.legend(loc='lower center', ncol=6, prop=legend_properties, bbox_to_anchor=(0.5,-0.1))\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(f'./Results/known_percentage_{bench_name}_{mode}.png', bbox_inches='tight')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_histogram_time_model_total(model_name_list, results_list_generation, \"General\", 'Generation')\n",
    "plot_histogram_time_model_total(model_name_list, results_list_mcqa, \"General\", 'Multi-choice QA')\n",
    "plot_histogram_time_model_total(model_name_list, results_list_mcqa, \"General\", 'True/False', changed_title=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making Plots with Time Invariant Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "domain = \"CommonSense\"\n",
    "\n",
    "mode = \"generation\"\n",
    "results_list_generation_cs = get_results_list_invariant(model_name_list=model_name_list,\n",
    "                 domain=domain,\n",
    "                 mode=mode)\n",
    "\n",
    "mode = 'QA'\n",
    "results_list_mcqa_cs = get_results_list_invariant(model_name_list=model_name_list,\n",
    "                 domain=domain,\n",
    "                 mode=mode)\n",
    "\n",
    "mode = 'TF'\n",
    "results_list_tf_cs = get_results_list_invariant(model_name_list=model_name_list,\n",
    "                 domain=domain,\n",
    "                 mode=mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_histogram_time_model_invariant(model_name_list, results_list_generation, results_list_mcqa, results_list_tf, bench_name, changed_title=True):\n",
    "\n",
    "    years = sorted(results_list_generation[0].keys())\n",
    "    # extract the year part\n",
    "    year_labels = [int(year.split('_')[-1]) for year in years]\n",
    "\n",
    "    categories = ['highly_known', 'maybe_known', 'weakly_known', 'unknown']\n",
    "    colors = ['#3C6495', '#7E9BC7', '#B5C3DD', '#FDE58E']\n",
    "\n",
    "    # Create the plot\n",
    "    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(22, 4.5))  # 두 개의 그래프를 한 줄에 배치\n",
    "\n",
    "\n",
    "    for ax, results_list in zip([ax1, ax2, ax3], [results_list_generation, results_list_mcqa, results_list_tf]):\n",
    "        \n",
    "        max_val = 0\n",
    "        min_val = 100\n",
    "        for model_name, results in zip(model_name_list, results_list):\n",
    "\n",
    "            data = []\n",
    "            for year in years:\n",
    "                counts = [results[year][category] for category in categories]\n",
    "                total = sum(counts)\n",
    "\n",
    "                knowns = 0\n",
    "                for category in categories:\n",
    "                    if \"highly_known\" in category:\n",
    "                        knowns += results[year][category]\n",
    "\n",
    "                knowns_percentage = knowns / total * 100\n",
    "                data.append(knowns_percentage)\n",
    "\n",
    "                if knowns_percentage > max_val:\n",
    "                    max_val = knowns_percentage\n",
    "                if knowns_percentage < min_val:\n",
    "                    min_val = knowns_percentage\n",
    "\n",
    "            # pointplot\n",
    "            ax.plot(year_labels, data, linestyle='-', linewidth=3,\n",
    "                    label=official_model_name[model_name] if ax == ax1 else \"\",\n",
    "                    color=official_model_color[model_name],\n",
    "                    alpha=1)\n",
    "\n",
    "            min_space = 35\n",
    "            if max_val - min_val <= min_space:\n",
    "                stride = min_space - (max_val-min_val)\n",
    "                min_val = int(min_val - stride / 2)\n",
    "                max_val = int(max_val + stride / 2)\n",
    "\n",
    "            ax.set_xticks(year_labels)\n",
    "            ax.set_xticklabels(year_labels, fontsize=15)\n",
    "\n",
    "            min_val = 3 if min_val < 3 else min_val\n",
    "\n",
    "            ax.set_ylim(min_val - 3, max_val + 3)\n",
    "            ax.set_yticklabels(ax.get_yticks(), fontsize=15)\n",
    "            ax.yaxis.set_major_formatter(mticker.FuncFormatter(lambda x, _: f'{int(x)}'))\n",
    "            ax.grid(True)\n",
    "            ax.margins(x=0.03)\n",
    "            ax.grid(False, axis='x')\n",
    "            ax.patch.set_facecolor('#f1f1f0')\n",
    "\n",
    "        fig.legend(loc='lower center', ncol=6, fontsize=16, bbox_to_anchor=(0.5,-0.1))\n",
    "\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    plt.savefig(f'./Results/known_percentage_{bench_name}.pdf', bbox_inches='tight')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_histogram_time_model_invariant(model_name_list, results_list_generation_cs, results_list_mcqa_cs, results_list_tf_cs, \"CommonSense\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ckcheck",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}