TripAdvisor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"src/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (arrow_dataset.py, line 144)",
     "output_type": "error",
     "traceback": [
      "Traceback \u001b[0;36m(most recent call last)\u001b[0m:\n",
      "  File \u001b[1;32m\"/home/mripoll/.local/lib/python3.5/site-packages/IPython/core/interactiveshell.py\"\u001b[0m, line \u001b[1;32m3326\u001b[0m, in \u001b[1;35mrun_code\u001b[0m\n    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \u001b[1;32m\"<ipython-input-3-81e84f1dcb47>\"\u001b[0m, line \u001b[1;32m1\u001b[0m, in \u001b[1;35m<module>\u001b[0m\n    from datasets.tripadvisor import TripAdvisorDataset\n",
      "\u001b[0;36m  File \u001b[0;32m\"/home/mripoll/.local/lib/python3.5/site-packages/datasets/__init__.py\"\u001b[0;36m, line \u001b[0;32m26\u001b[0;36m, in \u001b[0;35m<module>\u001b[0;36m\u001b[0m\n\u001b[0;31m    from .arrow_dataset import Dataset, concatenate_datasets\u001b[0m\n",
      "\u001b[0;36m  File \u001b[0;32m\"/home/mripoll/.local/lib/python3.5/site-packages/datasets/arrow_dataset.py\"\u001b[0;36m, line \u001b[0;32m144\u001b[0m\n\u001b[0;31m    self: \"Dataset\" = args[0]\u001b[0m\n\u001b[0m        ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "from datasets.tripadvisor import TripAdvisorDataset\n",
    "\n",
    "dataset = TripAdvisorDataset(\n",
    "        text_processor='word2vec', \n",
    "        text_processor_filters=['lowercase', 'stopwordsfilter'],\n",
    "        embedding_path='data/embeddings/word2vec/glove.6B.50d.txt',\n",
    "        data_path='data/'\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dawid-Skene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import models.dawid_skene as ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_input(dataset, female_model, male_model, missing_value=-1):\n",
    "    data = {}\n",
    "    for data_point in dataset:\n",
    "        annotator = data_point['annotator']\n",
    "        question = data_point['text']\n",
    "        annotation = data_point['label'].item()\n",
    "        \n",
    "        data[question] = {}\n",
    "        #data[question]['truth'] =  [ annotation ]\n",
    "        data[question]['female'] = [ female_model(data_point['embedding']).argmax().item() ]\n",
    "        data[question]['male'] =   [ male_model(data_point['embedding']).argmax().item() ]\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Pretrained Male, Female Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model_for_inference(path):\n",
    "    from models.basic import BasicNetwork\n",
    "    model = BasicNetwork(embedding_dim=50, label_dim=2)\n",
    "    model.load_state_dict(torch.load(path, map_location=torch.device('cpu')))\n",
    "    model.eval()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "male_model_path = 'models/tripadvisor_pad100/male_basic_0.89060_batch64_lr8.299248182022548e-05_20200925-075432.pt'\n",
    "female_model_path = 'models/tripadvisor_pad100/female_basic_0.89038_batch64_lr0.0003306989309627488_20200924-222149.pt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "male_model = load_model_for_inference(male_model_path)\n",
    "female_model = load_model_for_inference(female_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transform dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = prepare_input(dataset, female_model, male_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num Patients: 13707\n",
      "Observers: ['female', 'male']\n",
      "Classes: [0, 1]\n",
      "Iter\tlog-likelihood\tdelta-CM\tdelta-ER\n",
      "1 \t -14253.315573351305\n",
      "2 \t -14250.461496730466 \t0.000038\t0.025316\n",
      "3 \t -14250.418910985853 \t0.000005\t0.003141\n",
      "4 \t -14250.41821090512 \t0.000001\t0.000403\n",
      "5 \t -14250.41819926299 \t0.000000\t0.000052\n",
      "6 \t -14250.418199067128 \t0.000000\t0.000007\n",
      "Class marginals\n",
      "[0.45 0.55]\n",
      "Error rates\n",
      "[[[0.94 0.06]\n",
      "  [0.06 0.94]]\n",
      "\n",
      " [[0.92 0.08]\n",
      "  [0.05 0.95]]]\n",
      "Incidence-of-error rates\n",
      "[[0.42 0.03]\n",
      " [0.03 0.52]]\n",
      "[[0.41 0.04]\n",
      " [0.02 0.53]]\n"
     ]
    }
   ],
   "source": [
    "patients, observers, classes, counts, class_marginals, error_rates, patient_classes = ds.run(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[autoreload of six failed: Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/IPython/extensions/autoreload.py\", line 245, in check\n",
      "    superreload(m, reload, self.old_objects)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/IPython/extensions/autoreload.py\", line 410, in superreload\n",
      "    update_generic(old_obj, new_obj)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/IPython/extensions/autoreload.py\", line 347, in update_generic\n",
      "    update(a, b)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/IPython/extensions/autoreload.py\", line 287, in update_class\n",
      "    old_obj = getattr(old, key)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/six.py\", line 93, in __get__\n",
      "    setattr(obj, self.name, result)  # Invokes __set__.\n",
      "AttributeError: 'NoneType' object has no attribute 'cStringIO'\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import math\n",
    "\n",
    "def plot_error_matrices(error_rates, observers, class_marginals):\n",
    "    nrows, ncols = int(math.ceil(len(observers)/3)), 3\n",
    "    f, axs = plt.subplots(nrows=nrows, ncols=ncols, figsize=(ncols*5,nrows*5))\n",
    "    for i, obs in enumerate(observers):\n",
    "        error_rate = error_rates[i]\n",
    "        incidence_of_error_rate = class_marginals * error_rate\n",
    "        \n",
    "        sns.heatmap(error_rate, vmin=0, vmax=1, annot=True, cbar=False, ax=axs.flat[i])\n",
    "        axs.flat[i].set_title(f'Error rate \\n {obs}')\n",
    "        axs.flat[i].set_xlabel('Predicter label (Observer label)')\n",
    "        axs.flat[i].set_ylabel('Latent truth')\n",
    "    plt.show(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3oAAAFcCAYAAACeOcx6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd5hU5dnH8e+9C9goAtKkaWxR1BhBrIk9YoVoYjTVWIgajMYezWtNbInRGE0iiUZjYtcosYARa1QUBAugRESRjlIVUIF93j9mwN11y4DszO7h+7muvZhzzjNz7jO7e7O/OS1SSkiSJEmSsqOs1AVIkiRJktYsg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMelopIt6NiCUR8VGlr+tLXVd9IuKpiDi+1HVIajj2J0mSVk2zUhegRufQlNLj9Q2KiGYppWX1zVvV11idMZLWGvYnSZIK5B49FSQijomI5yLimoiYA1xUy7yyiPhlREyOiNkR8feIaJN/jU0iIkXEcRHxHvBEDevZKyKmRsQ5ETET+FtEtI2IhyLi/YiYl3/cLT/+18DXgOsrf8IfEV+OiP9ExNyImBARRxbrvZJUXPYnSZI+z6CnVbEzMAnoBPy6lnnH5L/2Br4EtASqH161J7A1cEAt6+kMtAN6AgPJ/Zz+LT/dA1iy4jVTSucDzwKDUkotU0qDImID4D/A7UBH4CjgjxGxzWpvuaTGzv4kSVIlBj1V90BEzK/0dUKlZdNTSn9IKS1LKS2pZd73gN+llCallD4CfgEcFRGVDxO+KKW0qNJrVFcBXJhS+iSltCSlNCeldF9KaXFK6UNyf7DtWcc2HAK8m1L6W76uMcB9wLdX4/2Q1HjYnyRJKpDn6Km6AXWcAzOlgHkbA5MrTU8m93PWqZ7Xqez9lNLHKyYiYn3gGqAf0DY/u1VElKeUltfw/J7AzhExv9K8ZsBt9axXUuNmf5IkqUAGPa2KVMC86eT+kFmhB7AMmAV0q+N16nrNM4CtgJ1TSjMjYgdgDBC1jJ8CPJ1S2r+e9UjKDvuTJEmVeOim1rQ7gJ9HxKYR0RK4DLjrC16ZrhW5817mR0Q74MJqy2eRO99mhYeALSPiBxHRPP+1U0Rs/QVqkNT02Z8kSWsNg56q+3e1+1T9axWffzO5Q5CeAd4BPgZO+YI1XQusB3wAjACGVlv+e+Bb+SveXZc/T+Yb5C5yMB2YCVwJrPMF65BUWvYnSZIKFCnVd5SKJEmSJKkpcY+eJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6OkLiYjdI+Kt/BXwBhRxvZtERIoI7wUpqSQi4qmIOL7UdUiSVBODnr6oS4DrU0otU0oPlLoYSZIkSQY9fXE9gXGlLkKSJEnSZwx6Wm0R8TbwJT67ifE6EdEmIm6KiBkRMS0ifhUR5fnxx0TEcxFxTUTMj4hJEbFbfv6UiJgdET+q9PoHR8SYiFiYX35RHbXUul5JWiF/yPfJ+UPOP4yISyNis4h4Pt9r7o6IFvmxbSPioYh4P3/D84ciolsdr31sRLyRHzssInoWb8skSarKoKfVllLaDHgPODR/6OYnwC3AMmBz4KvAN4DK57DsDLwGtAduB+4EdsqP/z5wfUS0zI9dBPwQ2BA4GDipjvMA61uvJK1wANAb2AU4GxhMrv90B7YFjs6PKwP+Ru7IhR7AEuD6ml4wIvoD5wGHAx2AZ4E7GmwLJEmqh0FPa0xEdAIOAk5LKS1KKc0GrgGOqjTsnZTS31JKy4G7yP1hdUlK6ZOU0mPAp+TCGimlp1JKr6eUKlJKr5H7o2nP1VyvJK1wVUppYUppHDAWeCylNCmltAB4lNyHRaSU5qSU7kspLU4pfQj8mhp6UN6JwOUppTdSSsuAy4Ad3KsnSSoVr1ioNakn0ByYEREr5pUBUyqNmVXp8RKAlFL1eS0BImJn4Apyn7C3ANYB7lnN9UrSCtV7TvXpzgARsT65D436AW3zy1tFRHn+w6rKegK/j4irK80LoCsweQ3WLklSQQx6WpOmAJ8AG+U/0f6ibid3mNSBKaWPI+JaYKMirFeSAM4AtgJ2TinNjIgdgDHkAlx1U4Bfp5T+WcwCJUmqjYduao1JKc0AHgOujojWEVGWv8hBbYc61acVMDcf8voC3y3SeiUJcj1oCTA/ItoBF9Yx9s/ALyKiF6y8QNS3i1CjJEk1MuhpTfshucMsxwPzgHuBLqv5WicDl0TEh8AFwN1FWq8kAVwLrAd8AIwAhtY2MKX0L+BK4M6IWEju3L8Di1GkJEk1iZRSqWuQJEmSJK1B7tGTJEmSpIwx6EmSJBVRRNwcEbMjYmwtyyMirouIiRHxWkTsWOwaJTV9Bj1JkqTiuoXcbTtqcyCwRf5rIPCnItQkKWMMepIkSUWUUnoGmFvHkP7A31POCGDDiPACY5JWiUFPkiSpcelK7t6MK0zNz5OkgjXaG6Z/8saTXg50LdFjl5NKXYKKZNaCN2u60XST8/HoIfantcBW+/+y1CWoiCbPea1J9qeIGEju8E422GCD3l/+8pdLXJGkNenll1/+IKXUYXWe22iDniRJ0lpqGtC90nS3/LzPSSkNBgYD9OnTJ40aNarhq5NUNBExeXWf66GbkiRJjcsQ4If5q2/uAixIKc0odVGSmhb36EmSJBVRRNwB7AVsFBFTgQuB5gAppT8DjwAHAROBxcCPS1OppKbMoCdJklREKaWj61megJ8WqRxJGeWhm5IkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkFVlE9IuICRExMSLOrWF5j4h4MiLGRMRrEXFQKeqU1HQZ9CRJkoooIsqBG4ADgW2AoyNim2rDfgncnVL6KnAU8MfiVimpqTPoSZIkFVdfYGJKaVJK6VPgTqB/tTEJaJ1/3AaYXsT6JGWAQU+SJKm4ugJTKk1Pzc+r7CLg+xExFXgEOKWmF4qIgRExKiJGvf/++w1Rq6QmyqAnSZLU+BwN3JJS6gYcBNwWEZ/7uy2lNDil1Cel1KdDhw5FL1JS42XQkyRJKq5pQPdK093y8yo7DrgbIKX0ArAusFFRqpOUCQY9SZKk4hoJbBERm0ZEC3IXWxlSbcx7wL4AEbE1uaDnsZmSCmbQkyRJKqKU0jJgEDAMeIPc1TXHRcQlEXFYftgZwAkR8SpwB3BMSimVpmJJTVGzUhcgSZK0tkkpPULuIiuV511Q6fF4YPdi1yUpO9yjJ0mSJEkZY9CTJEmSpIwx6EmSJElSxniOXpH8d/Q4rvzr3VRUVHD4/rtz3BH9qiyfPnsOF/zh78xb+BFtWq7PZT8/ls4btV25/KPFSxhwysXss/NXOG/g0cUuX/XYe989+NWV51NeXsY//34vf7jmL1WWt2jRnOtvvJLtd+jFvLnzGfjj05ny3jS69+jKsy89zNtvvQPAy6Ne5eyfXwRA8+bNufy3/8due/SloqKCyy+9loeHPFbsTdNa4LlX3uTKvw+hoqKCb+7dl+P671Nl+fT353HhjXd/1p9+ejSd2m/Im+9O49c3389Hiz+hvCw4/pv70m/XHUq0FarNnvvszoWXn0N5WRl3/uN+/vT7m6ssb9GiOb/746/Z7ivbMG/eAgYddxZTp0xnwLcOYuCgY1aO27rXlhy893cYP3YChww4gEGnn0B5eRnDH3uGKy6+tshbJUmqj0GvCJYvr+CyG+9g8MWn0ql9W44+63L26rs9m3XfeOWYq2+5j0P33oX+++zKi6+9yXW3PcBlP//xyuXX3z6E3ttsUYryVY+ysjKuuPoCjhxwLNOnzWLYk/cw7JEn+N+Et1eO+e4Pv8X8+QvZ5asHMOCIg/i/i89g4I9PB2DyO++x79e++bnXPe3ME/ng/Tns1rsfEUHbtm2Ktk1aeyyvqOCyv/2LG88bSKf2bfju+dexV+9ebNat08oxv/vnQxz6td4ctmcfXhw7kd/f+SiX/fRo1l2nBb866Sh6dunA7LkLOPr837Pb9lvReoP1SrhFqqysrIxLrzqP7x0xkJnTZzHk8Tt4fOhTvDVh0sox3/n+4SyYv5A9dzqEQ7/Zj3MvPI1Bx5/NA/c+wgP35q4VstXWW/CX265l/NgJbNi2DeddfDqH7HMUc+fM4+obfsXuX9+Z5555sVSbKUmqQYMduhkRX46IcyLiuvzXOfn7wKx1xr71Lj26dKRb5w40b96MfnvsxJMvvlZlzKQpM9h5u60A6LvdVjz50qsrl42fOJm58z9ktx3Wyrev0dux9/a8M+k9Jr87laVLl/LA/Y/Q7+B9q4zpd9C+3H37AwD8+4Fh7LHnrvW+7tHfP5zrfjcYgJQSc+fOX/PFr4XsTVWNnfge3TtvRLdO7WnerBn9dt2Bp0aNqzLm7amz6Lvt5gD07bUZT72cW75Jlw707NIBgI7t2tCudUvmLfyouBugOu2w47a8+857TJk8jaVLl/Hvfw1l/wP3rjJm/wP34r47c7dwe2TIf9j96zt/7nUOO+JA/v2voQD02KQb7056j7lz5gHw36dHcOCh+zXwlkiSVlWDBL2IOAe4EwjgpfxXAHdExLkNsc7GbNbceXSqdBhmp/YbMnvuvCpjttykG4+PGAPA8BGvsGjJx8xf+BEVFRX89m/3cvoxRxS1ZhWu88admD5txsrp6dNm0rlLpypjunTpyLT8mOXLl/Phwg9p125DAHr07Mbjz97Pvx6+jZ137Q1A6zatADjn/FP5zzP38Zdbr6VDh/bF2JxMszd93ux5C+ncfsOV0x3bt2HWvAVVxmzVswvDX3odgOEjx7JoySfM/3BRlTGvT3yPpcuW072TP6eNSecunZgxbdbK6RnTZ9G5S8fPjZk+PTcm158+om27DauMOXTAATx436MAvDvpPb60+SZ0674x5eXlHHDQPnTZuHMDb4kkaVU11KGbxwG9UkpLK8+MiN8B44ArGmi9TdYZPz6CywffyZAnRrBjr83p2H5DysrKuOvRp9mj97ZVztdTdsyaOZsde+3DvHnz2X6HXtzyz+v5+i6H0Ky8nK7dujDypTFceP4V/OSnx3Dhr85m0E/OKXXJTZ29aTWc/r1DuPyWB3jw6VH03vpLdGzXhrKyzz4nfH/eQs7/45386qTvVJmvbNih93YsWfIx/3tzIgALF3zI+Wf+iutv+g2pooKXX3qFHpt2L3GVkqTqGiroVQAbA5Orze+SX1ajiBgIDAS4/qLTOf7IQxqovOLq1K4tsz74bA/erDnz6diuanDr2G5Drjn3RAAWL/mYx18YQ+uW6/PqhEmMHj+Rux99msUff8LSZctZf911Oe2Hnz+nS6Uxc/osNu7aZeX0xl07M3PGrCpjZsyYTdeuXZgxfRbl5eW0at1q5aGYn36a+/e1V8bx7jtT2GzzTXl1zFgWL1q88uIr/35gKN/9gXt114DV6k1QrT+dfzLHHX5AgxRYbB3btmbmnM8OC549ZwGdqp0P2rFdG645/UcALP74Ex5/6fWV5+F9tPhjBl11M6d8px/bb9GzeIWrIDNnzKJL18+OMOiycSdmzpj9uTEbb9yJmSv7U0vmVTpU/NBv9mPI/Y9Wec7wYU8zfNjTABz9wyNYvrzOXx9JUgk0VNA7DRgeEW8BU/LzegCbA4Nqe1JKaTAwGOCTN55MDVRb0fXaoieTZ8xm6qwP6NRuQ4b+dyRXnH5clTErrmZXVlbGX+8byjf33Q2gyrgHhz/PuLcnG/IamTGjX+dLm/WkR8+uzJg+mwGHH8RJx59ZZcywR57gyO8OYNTIVzh0wAH895kRALRv35Z58xZQUVFBz0268aXNejL53dyvzGNDn2T3r/Xlv8+8yNf23LXKxV202larN0HV/vTx6CHZ6U+bdee9mR8wdfZcOrVrzdAXXuHyQd+tMmbewkW0abkeZWVl3PTgEwzYaycAli5bxs9/dyuHfq03+++8fSnKVz1eHTOOTb/Uk+49ujJzxiwO/WY/fjaw6lHKjw99iiOOOozRo17joMP25/lnX1q5LCI4ZMA3+NbBx1R5TvuN2jHng7m0btOKHxz7HX563FnF2BxJ0ipokKCXUhoaEVsCfYGu+dnTgJEppeUNsc7GrFl5Oeed8B1Ouvg6li+vYMB+u7F5j4254fYhbLN5T/bu+xVGjp3Adbc9QESw4zZbcP5Pjip12SrQ8uXL+cWZl3Ln/TdRXl7GHf+4jwlvTuTs807h1TFjGfbok9x+271cP/gqRowZxvx5C/jJsbkrbu6y+06cfd4pLFu6jIpUwdk/v4j5+fOjLr3waq6/8Uouvfw85syZy6knn1fKzcwEe9PnNSsv5xfHDOCky/9CRUUFA/bqy+bdO3PDPcPotWk39urTi1FvvM11d+b26PTe+kuc9+Pch03DXniV0W9OYsFHixjyzEgALjnxO3x5k661rk/FtXz5ci445zL+fs+fKC8v5+7bH+CtCW9z+rkn89or43l86FPc9Y9/cc2fLuPpkQ8xf/4CBh1/9srn77xbb6ZPm8WUydOqvO6Fl53DNttuCcDvf3Mj77xdfSe5JKnUIqXG+cF0lvboqW49djmp1CWoSGYteDNKXcOakKU9eqrdVvv/stQlqIgmz3mtyfenPn36pFGjRpW6DElrUES8nFLqszrP9ax5SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIyxqAnSZIkSRlj0JMkSZKkjDHoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSVKRRUS/iJgQERMj4txaxhwZEeMjYlxE3F7sGiU1bc1KXYAkSdLaJCLKgRuA/YGpwMiIGJJSGl9pzBbAL4DdU0rzIqJjaaqV1FS5R0+SJKm4+gITU0qTUkqfAncC/auNOQG4IaU0DyClNLvINUpq4urdoxcRWwJnAT0rj08p7dOAdUlSvexPkpqorsCUStNTgZ2rjdkSICKeA8qBi1JKQ4tTnqQsKOTQzXuAPwN/AZY3bDmStErsT5KyqhmwBbAX0A14JiK2SynNrzwoIgYCAwF69OhR7BolNWKFBL1lKaU/NXglkrTq7E+SmqJpQPdK093y8yqbCryYUloKvBMR/yMX/EZWHpRSGgwMBujTp09qsIolNTm1nqMXEe0ioh3w74g4OSK6rJiXny9JJWF/ktTEjQS2iIhNI6IFcBQwpNqYB8jtzSMiNiJ3KOekYhYpqWmra4/ey0ACIj99VqVlCfhSQxUlSfWwP0lqslJKyyJiEDCM3Pl3N6eUxkXEJcColNKQ/LJvRMR4coemn5VSmlO6qiU1NbUGvZTSpgARsW5K6ePKyyJi3YYuTJJqY3+S1NSllB4BHqk274JKjxNwev5LklZZIbdXeL7AeZJUbPYnSZKkGtS6Ry8iOpO7/O96EfFVPjtEqjWwfhFqk6Qa2Z8kSZLqVtc5egcAx5C7EtTvKs3/EDivAWuSpPrYnyRJkupQ1zl6twK3RsQRKaX7iliTJNXJ/iRJklS3Qu6jt21E9Ko+M6V0SQPUI0mrwv4kSZJUg0KC3keVHq8LHAK80TDlSNIqsT9JkiTVoN6gl1K6uvJ0RPyW3L1dJKmk7E+SJEk1K+T2CtWtT+4CCJLU2NifJEmSKGCPXkS8DqT8ZDnQAfD8F0klZ3+SJEmqWSHn6B1S6fEyYFZKaVkD1SNJq8L+JEmSVIM6g15ElAPDUkpfLlI9klQQ+5MkSVLt6jxHL6W0HJgQET2KVI8kFcT+JEmSVLtCDt1sC4yLiJeARStmppQOa7CqJKkw9idJkqQaFBL0/q/Bq5Ck1WN/kiRJqkEhQe+glNI5lWdExJXA0w1TkiQVzP4kSZJUg0Luo7d/DfMOXNOFSNJqsD9JkiTVoNY9ehFxEnAy8KWIeK3SolbAcw1dmCTVxv4kSZJUt7oO3bwdeBS4HDi30vwPU0pzG7QqSaqb/UmSJKkOtQa9lNICYAFwdPHKkaT62Z8kSZLqVsg5epIkSZKkJsSgJ0mSJEkZU+/tFSLiypouX1593pq2ya4/bciXVyPy3nN/KHUJaqJK1Z823vOMhnx5NRIzRt9a6hIkSVpt3l5BUlNmf5IkSaqBt1eQ1OTYnyRJkurm7RUkNUX2J0mSpDoUdHuFiCgHOuXHt4yIliml94pUoyRVYX+SJEmqWyEXYxkEXATMAirysxOwfcOVJUn1sz9JkiTVrN6gB5wGbJVSmtPQxUjSKrI/SZIk1aCQq25OIXeIlCQ1NvYnSZKkGhSyR28S8FREPAx8smJmSul3DVaVJBXG/iRJklSDQoLee/mvFvkvSWos7E+SJEk1qDfopZQuBoiI9VNKixu+JEkqjP1JkiSpZvWeoxcRu0bEeODN/PRXIuKPDV6ZJNXD/iRJklSzQi7Gci1wADAHIKX0KvD1hixKkgpkf5IkSapBIUGPlNKUarOWN0AtkrTK7E+SJEmfV8jFWKZExG5AiojmwKnAGw1bliQVxP4kSZJUg0L26J0I/BToCkwDdgBObsiiJKlA9idJkqQaFLJHb6uU0vcqz4iI3YHnGqYkSSqY/UmSJKkGhezR+0OB8ySp2OxPkiRJNah1j15E7ArsBnSIiNMrLWoNlDd0YZJUG/uTJElS3eo6dLMF0DI/plWl+QuBbzVkUZJUD/uTJElSHWoNeimlp4GnI+KWlNLkItYkSXWyP0mSJNWtkIuxLI6I3wC9gHVXzEwp7dNgVUlSYexPkiRJNSjkYiz/BN4ENgUuBt4FRjZgTZJUKPuTJElSDQoJeu1TSjcBS1NKT6eUjgX8tFxSY2B/kiRJqkEhQW9p/t8ZEXFwRHwVaNeANUlSoexPkpqkiOgXERMiYmJEnFvHuCMiIkVEn2LWJ6npK+QcvV9FRBvgDHL3p2oNnNagVUlSYexPkpqciCgHbgD2B6YCIyNiSEppfLVxrYBTgReLX6Wkpq7ePXoppYdSSgtSSmNTSnunlHoDmxWhNkmqk/1JUhPVF5iYUpqUUvoUuBPoX8O4S4ErgY+LWZykbCjk0M2anF7/EEkqCfuTpMauKzCl0vTU/LyVImJHoHtK6eFiFiYpO1Y36MUarUKS1hz7k6QmLSLKgN+ROyy9vrEDI2JURIx6//33G744SU3G6ga9tEarkKQ1x/4kqbGbBnSvNN0tP2+FVsC2wFMR8S6wCzCkpguypJQGp5T6pJT6dOjQoQFLltTU1Hoxloj4kJr/YApgvQarSJLqYX+S1MSNBLaIiE3JBbyjgO+uWJhSWgBstGI6Ip4CzkwpjSpynZKasFqDXkqpVTELkaRC2Z8kNWUppWURMQgYBpQDN6eUxkXEJcColNKQ0lYoKQsKub2CJEmS1qCU0iPAI9XmXVDL2L2KUZOkbFndc/QkSZIkSY2UQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGvQa097578OzIh3l+9FAGnXb855a3aNGcP998Nc+PHsrDj99Jtx4br1y2da8t+fdjt/PUC0N44rkHWGedFlWee8sd1/Pk8w82+DZo1f13zHgOPeUSDv7pRdx0/2OfWz599lyOv+g6jvj5ZRx7wbXMnDOvyvKPFi9hvxN+yWV/ubtYJWsttO9+X+PF0cMY9crjnHr6wM8tb9GiBTfdci2jXnmc/zxxL917dK2yvGu3Lrw34xUG/ey4lfNat2nFLbf9gREvD2XEqKHs1HeHBt8OrZr/vvw6h/7kFxx8wjncdM/Dn1s+ffYHHH/eVRwx6P849twrmPnB3JXLdjjsWL59ygV8+5QLOOWS3xezbEnSamhW6gKyqqysjMt++0u+M+B4ZkyfxaNP3sVjjz7J/ya8vXLM0T84ggXzF7Lbjv3of/iB/PKiMzjx2DMoLy/n+sFXcspPzmX82Am0bduGpUuXrXzeQYfux6KPFpdis1SP5csruOwvdzP4gkF0ar8hR5/zG/baaTs2695l5Zir//4vDt2zL/333oUXX5/Adf8YwmWn/mjl8uvveJje22xWivK1ligrK+Oqqy/i8P7HMH3aTIY/fR9DH36CCRMmrhzz/R9+i/nzF9Jnh/04/IiDueiSszjumNNWLv/15ecx/D/PVHndy6/6JcMff4ZjfnAKzZs3Z7311y3aNql+y5K7VyYAABVySURBVJdXcNmfbmPwr86kU/t2HP3zS9hr5x3YrFKIv/qmuzh0393ov+8evPjqeK679V4uOyP3QcA6LVpwzx8uKVX5kqRV5B69BvLV3tvx7qT3eG/yVJYuXcqD9z3KAQftU2VMv4P24e47HgDgoQcf42t77gLAnvvszhtj/8f4sRMAmDdvARUVFQCsv8H6/OTkH/H7395YxK1RocZOfJcenTeiW+eNaN68Gf322JEnR75WZcykKTPYebutAOi77ZY8OfL1lcvGv/0ecxcsZLevbF3UurV26d1ne96ZNJnJ705h6dKl3H/fwxx4yL5Vxhx08H7cefv9ADz4wFC+vteuny07ZD8mT57Km2+8tXJeq9Yt2W23nbjt1nsAWLp0KQsXfFiErVGhxv5vEj26dKRb5465/vT1vjw5YkyVMZOmTGfn7XP9p+/2W39uuSSp6Sh60IuIHxd7naXQuUsnpk2buXJ6xvSZdO7S8XNjpufHLF++nIULP6Rduw3ZbPOeJBJ33DeYx56+l5N/duzK55xz/in8+YZbWLxkSXE2RKtk1twFdNqo7crpTu3aMnvOgipjttykK4+PeAWA4S++yqIlHzP/w4+oqKjgt7fez+k/+mZRa9Zn1pb+1KVLZ6ZNm7Fyevq0mXTp0qnqmI07MW1qpf604CPatW/LBhusz6k/H8hVl/+hyviePbvzwQdzuf7PV/LUfx/k99f/mvXXX6/hN0YFmzVnHp06tFs53Wmjdsyuduj4lpt25/HnXwZg+Asv5/rTwo8A+PTTpRx12sV874xLeeKF0cUrXJK0WkqxR+/iEqyzSSkvb0bfXXbkpyecTf9+3+fAQ/Zjj6/vQq/tvkzPTbvz6EPDS12ivoAzfvRNXh4/kSPPvIJR4ybSsd2GlJWVcdfQZ9ljx150bt+2/hdRQ7E/1eOc807hT9f/jUWLqh4+3qxZOV/ZoRd/++vt7LVHfxYvWsJpp/+kRFVqdZ1x7Hd4eewEjvzZhYx6fQId27elrCz3p8LQm3/LnddeyJVn/YSr/nI7U2bMLnG1kqS6NMg5ehHxWm2LgE61LCMiBgIDAVqv15n1WzTdP3hnzphF166dV0532bgzM6v9pzhzxiw27tqZGdNnUV5eTuvWrZg7dz4zps9kxPOjmDt3PgBP/OcZtvvKNixatIiv7LAtL732H8rLy9moQ3vue+gWjjjkmGJumurQqV0bZn3w2Sfks+bOo2P7NlXGdGy3IdecfQIAi5d8wuMjXqH1Buvz6v/eYfQbb3P30GdZ/PEnLF22nPXXXYfTftC/qNuQdWuiP62/TgfWad6mtqGN3owZM+na9bPzRjfu2pkZM2ZVHTN9Fl27dWb69Jm5/tSmJXPnzKN3n69wWP9+XHTp2bRp05qKigo+/vgThjwwlOnTZvLyqFcBePDBoQa9RqZT+7bMev+zi6vM+mAuHat9sNSxfVuuOf8UABYv+ZjHn3+Z1i3Xzz0/f7RCt84d6bPdl3nj7cl0r3akiiSp8Wioi7F0Ag4A5lWbH8DztT0ppTQYGAzQZcNtUgPVVhSvjB7Lppv1pHvPrsycPpv+RxzIycefXWXMsEef5MijB/DyyFc5pP83+O8zLwLw1PDn+Ompx7Heeuvy6adL2WX3nRj8x1sZ/tgz/P3muwDo1mNjbrvzT4a8RqbX5j2ZPON9ps76gE7tNmTof0dzxWnHVBkzb+FHtGm5PmVlZfz1/mF8c5/cuZmVxz34xAjGvf2eIa9hfOH+1K7VFk26P41++XW+tNkm9OjZjRnTZ3H4EQcz8NjTq4x59JHhHPXdwxn50iv0H9CPZ58eAcDBB3x35ZhzfnEKixYt5q+D/wHAtGkz2HyLTZn41jvsueeuTHhzImo8em25KZOnz2bqzPfp1L4tQ595iSvOqhrG5y34kDatNsj1p3se5pv7fw2AhR8tYt11WtCieXPmLfiQV8a/xY+POLAUmyFJKlBDBb2HgJYppVeqL4iIpxponY3K8uXLOe+sX3PHfX+hvLyMO//xL/735kTOOm8Qr44Zx2OPPskdt93HH268kudHD2X+vPmceOyZACxYsJAbb7iVR5+4m5QSw//zDMMfe6aeNaoxaFZeznnHH8lJl97A8orEgH12YfMeXbjhjofYZvMe7L3T9owc9xbX/WMIEbDjNptz/glHlrrstY39aflyzj7zYu594GbKy8r552338uabE/nF+acyZszrDH3kCf7x93v4819+y6hXHmfevPkc/+Of1/u655x5KTf+9WpatGjOu+9OYdBJ5xZha1SoZuXlnHfi9zjpgqtZXlHBgP2/xuY9u3LDP/7FNltswt47f5WRr7/JdbfeS0Sw47Zbcv5JPwByF2m55PpbKYsyKlIFx3774CpX65QkNT6RUuP8YLqp79FT4d79r/djWluss+3+Ueoa1oSmvkdPhZkx+tZSl6AiWmeL3Zp8f+rTp08aNWpUqcuQtAZFxMsppT6r81xvryBJkiRJGWPQkyRJkqSMMehJkiRJUsYY9CRJkiQpYwx6kiRJkpQxBj1JkiRJyhiDniRJUpFFRL+ImBAREyPiczedjIjTI2J8RLwWEcMjomcp6pTUdBn0JEmSiigiyoEbgAOBbYCjI2KbasPGAH1SStsD9wJXFbdKSU2dQU+SJKm4+gITU0qTUkqfAncC/SsPSCk9mVJanJ8cAXQrco2SmjiDniRJUnF1BaZUmp6an1eb44BHa1oQEQMjYlREjHr//ffXYImSmjqDniRJUiMVEd8H+gC/qWl5SmlwSqlPSqlPhw4dilucpEatWakLkCRJWstMA7pXmu6Wn1dFROwHnA/smVL6pEi1ScoI9+hJkiQV10hgi4jYNCJaAEcBQyoPiIivAjcCh6WUZpegRklNnEFPkiSpiFJKy4BBwDDgDeDulNK4iLgkIg7LD/sN0BK4JyJeiYghtbycJNXIQzclSZKKLKX0CPBItXkXVHq8X9GLkpQp7tGTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJEmSJGWMQU+SJEmSMsagJ0mSJEkZY9CTJEmSpIwx6EmSJElSxhj0JEmSJCljDHqSJEmSlDEGPUmSJEnKGIOeJElSkUVEv4iYEBETI+LcGpavExF35Ze/GBGbFL9KSU2ZQU+SJKmIIqIcuAE4ENgGODoitqk27DhgXkppc+Aa4MriVimpqTPoSZIkFVdfYGJKaVJK6VPgTqB/tTH9gVvzj+8F9o2IKGKNkpo4g54kSVJxdQWmVJqemp9X45iU0jJgAdC+KNVJyoRmpS6gNjPmj18rP7WKiIEppcGlrkMNz+910zX3w7fWuv7kz+vaw+910xIRA4GB+clPImJsKetZAzYCPih1EV9QFrYBsrEdWdiGrVb3iY026K3FBgL+B7t28HutpsSf17WH3+uGNw3oXmm6W35eTWOmRkQzoA0wp/oL5UP5YICIGJVS6tMgFReJ29B4ZGE7srINq/tcD92UJEkqrpHAFhGxaUS0AI4ChlQbMwT4Uf7xt4AnUkqpiDVKauLcoydJklREKaVlETEIGAaUAzenlMZFxCXAqJTSEOAm4LaImAjMJRcGJalgBr3Gx8Nl1h5+r9WU+PO69vB7XQQppUeAR6rNu6DS44+Bb6/iy2bhe+c2NB5Z2I61ehvCowAkSZIkKVs8R0+SJEmSMsag10hERL+ImBAREyPi3FLXo4YTETdHxOwMXAJbawn709rB3tR01Pc7GRHrRMRd+eUvRsQmxa+ybgVsw+kRMT4iXouI4RHRsxR11qXQ3hgRR0REiohGd/XHQrYhIo7Mfy/GRcTtxa6xEAX8PPWIiCcjYkz+Z+qgUtRZm/r6b+Rcl9++1yJix0Je16DXCEREOXADcCCwDXB0RGxT2qrUgG4B+pW6CKkQ9qe1yi3Ymxq9An8njwPmpZQ2B64BrixulXUrcBvGAH1SStsD9wJXFbfKuhXaGyOiFXAq8GJxK6xfIdsQEVsAvwB2Tyn1Ak4reqH1KPB78Uvg7pTSV8ld2OiPxa2yXrdQd/89ENgi/zUQ+FMhL2rQaxz6AhNTSpNSSp8CdwL9S1yTGkhK6RlyV1CTmgL701rC3tRkFPI72R+4Nf/4XmDfiIgi1liferchpfRkSmlxfnIEuXsNNiaF9sZLyQXtj4tZXIEK2YYTgBtSSvMAUkqzi1xjIQrZjgS0zj9uA0wvYn31KqD/9gf+nnJGABtGRJf6Xteg1zh0BaZUmp6anydJpWZ/khqXQn4nV45JKS0DFgDti1JdYVa1rxwHPNqgFa26erchf3hd95TSw8UsbBUU8n3YEtgyIp6LiBER0Rj3+heyHRcB34+IqeSudntKcUpbY1br/2JvryBJkqRGKSK+D/QB9ix1LasiIsqA3wHHlLiUL6oZucMF9yK3V/WZiNgupTS/pFWtuqOBW1JKV0fEruTuUbltSqmi1IU1JPfoNQ7TgO6Vprvl50lSqdmfpMalkN/JlWMiohm5Q9XmFKW6whTUVyJiP+B84LCU0idFqq1Q9W1DK2Bb4KmIeBfYBRjSyC7IUsj3YSowJKW0NKX0DvA/csGvMSlkO44D7gZIKb0ArAtsVJTq1ozV+r/YoNc4jAS2iIhNI6IFuZNEh5S4JkkC+5PU2BTyOzkE+FH+8beAJ1LjunFyvdsQEV8FbiQX8hrjeWF1bkNKaUFKaaOU0iYppU3InWd4WEppVGnKrVEhP0sPkNubR0RsRO5QzknFLLIAhWzHe8C+ABGxNbmg935Rq/xihgA/zF99cxdgQUppRn1PMug1Avnj5wcBw4A3yF0VaFxpq1JDiYg7gBeArSJiakQcV+qapNrYn9Ye9qamobbfyYi4JCIOyw+7CWgfEROB04FGdVuUArfhN0BL4J6IeCUiGtUHTAVuQ6NW4DYMA+ZExHjgSeCslFJj2jtc6HacAZwQEa8CdwDHNKYPP2rqvxFxYkScmB/yCLmAPRH4C3ByQa/biLZRkiRJkrQGuEdPkiRJkjLGoCdJkiRJGWPQkyRJkqSMMehJkiRJUsYY9CRJkiQpYwx6a1hELM9fBnhsRNwTEet/gdfaKyIeyj8+LCJqvTxyRGwYEQVdarWedT5V3808I+Ld/L1UCn3NYyLi+lqWDYiICypND4yIN/NfL0XEHqu73lJo6PcvIgZFxLFftE6tnexPNY63P1UdY3+SpIww6K15S1JKO6SUtgU+BU6svDB/o8NVft9TSkNSSlfUMWRDCrynxhetZQ07G/hjvp5DgJ8Ae6SUvkzuvbs9Ijo3xIojotkXfH4p3r+bgVOKvE5lh/1p1difVo39SZIakVL/J5p1zwKbR8QmETEhIv4OjAW6R8Q3IuKFiBid/2S9JUBE9Mt/WjwaOHzFC1X71LRTRPwrIl7Nf+0GXAFslv+0/jf5cWdFxMiIeC0iLs7P+1wttRUfEX+KiFERMW7F8ys5OyJez3+qvXl+fIeIuC+/zpERsXtdb05EbAl8klL6ID/rHHI34vwAIKU0GrgV+Gk96/12fg/FqxHxTH5eeUT8ptL2/yQ/f6+IeDZyN14dHxFXRMTK14+IiyLizMb6/qWUFgPvRkTfut5bqQD2pzrYn+xPktTUGfQaSOQ+jT0QeD0/awvgjymlXsAi4JfAfimlHYFRwOkRsS65u90fCvQGavuk+Drg6ZTSV4AdgXHAucDb+U/rz4qIb+TX2RfYAegdEV+vXktKaXIdm3F+SqkPsD2wZ0RsX2nZgpTSdsD1wLX5eb8Hrkkp7QQcAfy1nrdpd2B0pelewMvVxozKz69rvRcAB+Tfj8Py847Lj90J2Ak4ISI2zS/bETg1pbQlcBdwZKXXPxK4q5G/f6OAr9WxXqlO9if7E/YnScq8L3RoiGq0XkS8kn/8LHATsDEwOaU0Ij9/F2Ab4LmIAGgBvAB8GXgnpfQWQET8AxhYwzr2AX4IkFJaDiyIiLbVxnwj/zUmP92S3B8A71WrpS5HRsRAcj8nXfI1v5Zfdkelf6/JP94P2Ca/TQCtV+wJqEUX4P0C6qispvU+B9wSEXcD9+fnfQPYPiK+lZ9uQ277PwVeSim9A5BSGhMRHSNiY6ADMC+lNCUiTqXxvn+zyf2sSKvK/mR/WsH+JEkZZ9Bb85aklHaoPCP/H+OiyrOA/6SUjq42rsrzvqAALk8p3VhtHZtUq6XmJ+c+XT4T2CmlNC8ibgHWrTQk1fC4DNglpfRxtdeqbTVLyP2Bs8J4cnsKnqg0rze5PQK1rjeldGJE7AwcDLwcEb3Jbf8pKaVh1WrZi89v/z3At8jtobhrxVAa7/u3Lrn3TlpV9if7k/1JktYSHrpZGiOA3Sud+7BB/nyQN4FNImKz/Lija3n+cOCk/HPLI6IN8CHQqtKYYcCx8dm5NV0jouMq1Nia3B8MCyKiE7nDvCr7TqV/X8g/foxKJ+IX8IfhG8DmlaavAq6MiPaVnn8M+Ysh1LbeiNgspfRiSukCcp/Adye3/SdFRPP8mC0jYoNa6rgLOIrcH1P35Oc15vdvS3Ln30gNwf6UY3+yP0lSk+YevRJIKb0fEccAd0TEOvnZv0wp/S9/KM3DEbGY3KFVrWp4iVOBwRFxHLAcOCml9EJEPBcRY4FH8+fBbA28kP/E9SPg+/nxhdT4akSMIffH3RRyhx9V1jYiXgM+4bM/+H4G3JCf3wx4hmpX9avmGeDqiIiUMyQiugLPR0Qi98fh91NKM+pZ728iYgtyn3IPB14ldwjSJsDoyL0B7wMDatnWcRHRCpi2Yl0ppcca8fu3O3BRIXVIq8r+tJL9yf4kSU1apJTqHyU1kIj4PfDvlNLjpa6lKYiIrwKnp5R+UOpapKyzP60a+5MkNS4euqlSuwxY7Zs2r4U2Av6v1EVIawn706qxP0lSI+IePUmSJEnKGPfoSZIkSVLGGPQkSZIkKWMMepIkSZKUMQY9SZIkScoYg54kSZIkZYxBT5IkSZIy5v8BSxlf3zmGcuIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x360 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_error_matrices(error_rates, observers, class_marginals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}