ver1_3_0922.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2019/09/22 ver.1.3\n",
    "\n",
    "## 一、文件内容\n",
    "特征选择：GBRT+RT+XGboost取平均值 选择前380个特征<br />\n",
    "单模型构建：<br />\n",
    " 1：SVR<br />\n",
    " 2：RF<br />\n",
    " 3：XGBoost<br />\n",
    " 4：KNN<br />\n",
    " 5：GBRT<br />\n",
    "Stacking：<br />\n",
    " XGboost默认\n",
    "## 二、数据集\n",
    "### 导入数据集变量<br />\n",
    "元数据集 训练集全部：dg_train →训练集 特征数据：dg_train<br />\n",
    "元数据集 测试集全部：dg_test  →测试集 特征数据：dg_test<br />\n",
    "<br />\n",
    "训练集 得分数据：score_train<br />\n",
    "测试集 序号：id_test<br />\n",
    "训练集 序号：id_train\n",
    "<br />\n",
    "### 处理数据集变量<br />\n",
    "标准化处理后的原训练集 特征数据：dg_scaled_train<br />\n",
    "标准化处理后的测试集合 特征数据：dg_scaled_test<br />\n",
    "### 特征选择结果\n",
    "特征选择后的训练集 特征数据：X<br />\n",
    "特征选择后的测试集 特征数据：X_predict<br />\n",
    "### 在X中继续划分数据<br />\n",
    "X_train：0.8比例的原训练集特征 用作训练和验证<br />\n",
    "X_test：0.2比例的原训练集特征 用作测试（模型选择）<br />\n",
    "y_train：0.8比例的原训练集分数 对应X_train<br />\n",
    "y_test：0.2比例的原训练集分数 对应X_test<br />\n",
    "\n",
    "## 三、模型信息\n",
    "\n",
    "\n",
    "变量名|模型名|超参数设置|随机数种子|训练集上预测值\n",
    ":---------:|:------------------:|:-----------------:|:-----------------:|:-------------------------:\n",
    "model_svc|支持向量回归||NA|\n",
    "model_svcRF|随机森林回归||160|\n",
    "model_xgbr|Xgboost回归||161|\n",
    "model_k_neighbor|KNN回归||NA|\n",
    "model_gradient_boosting_regressor|GBRT||162|\n",
    "\n",
    "## 四、随机数信息\n",
    "位置|使用函数|随机数种子\n",
    ":------:|:-------:|:--------:|\n",
    "特征处理|GBRT|20\n",
    "特征处理|train_test_split|21\n",
    "stacking|KFold|22\n",
    "预测|train_test_split|23\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1、前処理（preprocess）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#预设导入\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import joblib\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#机器学习导入\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import svm\n",
    "from sklearn import linear_model\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import explained_variance_score \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBRegressor\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn import ensemble\n",
    "from sklearn import neighbors\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dg_train=pd.read_csv('train.csv')\n",
    "dg_test=pd.read_csv(\"test.csv\")\n",
    "\n",
    "#单独提取ID列和score列\n",
    "id_train=dg_train[\"ID\"].values\n",
    "id_test=dg_test[\"ID\"].values\n",
    "score_train=dg_train[\"Score\"].values\n",
    "\n",
    "del dg_train[\"ID\"]\n",
    "del dg_train[\"Score\"]\n",
    "del dg_test[\"ID\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StandardScaler(copy=True, with_mean=True, with_std=True)\n",
      "[5.82623261e-04 4.17374998e+00 7.18538490e-01 ... 1.87939626e+00\n",
      " 9.97324521e-01 1.03583133e+00]\n",
      "[7.27939627e-04 7.21597472e-02 1.39518638e-02 ... 2.31160294e+00\n",
      " 6.76420780e-07 3.08326932e+00]\n",
      "[[-0.02159435 -0.43089696  0.8928487  ... -0.96316788 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435 -0.23359638  1.78179021 ...  1.38095879 -0.39457952\n",
      "   1.11859545]\n",
      " [-0.02159435 -0.12936211  0.96904369 ... -0.67771587  2.03718504\n",
      "  -0.58990668]\n",
      " ...\n",
      " [-0.02159435  1.25174197  0.83358594 ...  0.55025397  0.82130276\n",
      "  -0.58990668]\n",
      " [-0.02159435  1.17728892 -0.25007609 ... -1.39923858  0.82130276\n",
      "  -0.58990668]\n",
      " [-0.02159435 -0.48301409 -0.45326272 ...  0.18061334 -0.39457952\n",
      "  -0.58990668]]\n",
      "[[-0.02159435 -0.42717431 -0.74957655 ...  0.41476292  0.82130276\n",
      "  -0.58990668]\n",
      " [-0.02159435 -1.50674353  0.78278928 ... -0.40936467 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435  1.90320616  0.7150604  ...  0.67851006  0.82130276\n",
      "  -0.58990668]\n",
      " ...\n",
      " [-0.02159435 -1.04513462  1.66326468 ... -1.17561258 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435  0.41042251 -0.65644935 ... -0.00815331 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435 -0.73987712  0.12243273 ... -0.52512401  0.82130276\n",
      "  -0.58990668]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(13732, 3805)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#z-score 标准化\n",
    "scaler=StandardScaler().fit(dg_train)#标准化的mean var\n",
    "print(scaler)\n",
    "print(scaler.mean_)\n",
    "print(scaler.var_)\n",
    "dg_scaled_train=scaler.transform(dg_train)#标准化结果向量\n",
    "print(dg_scaled_train)\n",
    "np.shape(dg_scaled_train)\n",
    "\n",
    "dg_scaled_test=scaler.transform(dg_test)\n",
    "print(dg_scaled_test)\n",
    "np.shape(dg_scaled_test)#相同标准 标准化测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x262a1c62908>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3Tc1Zn/8fczo957G0mWbMu9Wy7YhBCqCcFAQhJDSAjJLluSkOwmm2V/v93sLjn5nV3gZJPskiyEFgIGHALYhGI6brjItuQuS1YvliWrWn1m7u8Py0QY2R7LM/pOeV7n+KDRfD36ANKjO/d773PFGINSSqnAZ7M6gFJKKe/Qgq6UUkFCC7pSSgUJLehKKRUktKArpVSQCLPqC6elpZmCggKrvrxSSgWk3bt3txlj0sd6zrKCXlBQQElJiVVfXimlApKI1J7rOZ1yUUqpIKEFXSmlgoQWdKWUChJa0JVSKkhoQVdKqSChBV0ppYKEFnSllAoSWtCVUipIaEFXSqkgYdlOUaWCwdoddZ/63B3L8i1IopSO0JVSKmhoQVdKqSChBV0ppYKEFnSllAoSWtCVUipIaEFXSqkgoQVdKaWChK5DV8rLxlqbDro+XfmejtCVUipIaEFXSqkgoQVdKaWChM6hK3WJhpxujrWeItxuIzkmnKSYCOw2sTqWCkFa0JUap8bOftaXNlLW0MnAsPvjzydFh7N6fg4zshMsTKdCkRZ0pcZhW2Ub31m7h54BJ3MciSzKT8ZuE9p7B9lc0cbT22uZnZPAlxblEhVutzquChFa0JW6SE9ureanfzrElPQ47l5RSFp85MfPFabFMj8viS0VbbxzuIWnttVw94oCIrWoqwmgN0WVugi/3VTFv796iGtmZvLyd1Z+opifEWazceX0DNYsyaeho4+nttUwOOyyIK0KNTpCV8pDT39Uw89eP8yNc7P55ZoFhNnPPx6a40jkq+Tzwq461u6s4xsrCj51s1QPyFDe5NEIXURWiUi5iFSKyH3nue42ETEiUuy9iEpZ74Vddfxk/UGunZXJLzwo5mfMdSRy83wHFSdO8cCbR3ycUoW6C35XiogdeBi4AZgF3C4is8a4Lh64F9jh7ZBKWenlvQ3c99J+Pjstnf+5YyHhHhbzM5YUprCsMIVHNlWxvrTRRymV8myEvhSoNMZUGWOGgOeBm8e47qfAA8CAF/MpZanX9jXzw3VlXDY5lUe+vpjIsPHd3LxxXjZLCpL5xz/uo6Klx8splTrNk4LuAOpHPW4Y+dzHRGQhkGeM+dP5XkhE7hGREhEpaW1tveiwSk2k9aWN3Pv8XhZPSuaxu4ovaflhmM3Gw3csIiYijO89t5cBvUmqfMCTm6JjbXkzHz8pYgP+C/jmhV7IGPMo8ChAcXGxucDlSlnmpT0N/OgPZSwpSOGJby4hJiLsnF0UPZWREMVDX57Ht54q4T/eOMK/rZ7tpbRKnebJCL0ByBv1OBdoGvU4HpgDfCAiNcByYIPeGFWB6okt1fzwD2VcNiWVp+5eSmyk9xaDXTUjk7tXFvDUthreP3LCa6+rFHg2Qt8FFIlIIdAIrAHuOPOkMaYLSDvzWEQ+AH5kjCnxblSlvG/0qNttDBsPHGdzZRurZmfxizULfLLL874bZrC1so1/emk/91wxWXeSKq+5YEE3xjhF5LvARsAOPGGMOSgi9wMlxpgNvg6plK853W5e2tNIaX0nyyencHlRGi/t8c2KlMgwOw/eNp9bf72VNw40c+vCXJ98HRV6PHovaYx5HXj9rM/95BzXXnnpsZSaOIPDLp7dWUfliVNcNyuTz05LR8S33RLn5yXxl1dM5pEPq5jrSGJqRpxPv54KDbr1X4W0noFhfruliqrWU3xpUS5XTs/weTE/4++umUZaXASvlDYy7HJf+C8odQFa0FXIqmnr5ZFNVbT2DHLn8kksnpQ8oV8/KtzO6vkO2nuH2FbZNqFfWwUnLegqJB1u7uZLv9nGwLCLv7h8MjOyrOldPjUjjpnZCbxf3kp3/7AlGVTw0IKuQs7Rlh6+9tgOwu02/uqKKeSlxFia5/NzsnAZw1uHjluaQwU+7baoQsbaHXWc6Bngsc3ViMA3VxSQFvfp9rcTLTUukpVT0thU0cqKKWkX/gtKnYOO0FXI6Oof5smtNRjg25cX+kUxP+PK6elEhdt493CL1VFUANOCrkJCV98wT22rZmDYxd0rCsiIj7I60idEhdu5fGoah4/3cKCxy+o4KkDplIsKeoNOF3/5+xLaeob45soCcpKiLclxoV4wK6aksaWyjV+8U8Fjd2nnDHXxtKCroGaM4Z/+uJ+d1e18tTiPKen+u4HnzCj9ncMtPLixHMeoXzx6ipHyhE65qKD26w+O8dLeRv7+2mnMz0uyOs4FrZiSRlS4jQ/KtXGXunha0FXQevPAcR7cWM7NC3L43lVTrY7jkahwO8sKUznU1E1775DVcVSA0YKuglLliVP8cF0p8/OS+M8vzZuw7fzesHxyKiLw0THdPaoujs6hq6DTMzDM7Y9uB2DV7CyfdU30lcTocOY6Eimp7eDqmZnaXld5TEfoKqgYY/jxi/s42TvI7UvzSYwOtzrSuKycmsag001JbYfVUVQA0YKugsranXW8ceA4183KYrIfr2i5kNzkGApSY9h2rA230dMalWd0ykUFtNFru1u6B/j1B5UUZcRxeVHgb6FfMSWNtTvrKD/eY3UUFSB0hK6CgtPlZl1JPRF2G7ctzsUWQDdBz2VmdgLxUWHsqD5pdRQVILSgq6DwfnkrzV0DfGlxLvFRgTlvfja7TVhSkEJFyynq2/usjqMCgBZ0FfBaugfYdLSVhXlJlvU195UlBSmIwLMXaBugFGhBVwHObQyv7G0kMtzGDXOzrY7jdYnR4czISmBdST2DTpfVcZSf04KuAlpJTQe17X18fk42cZHBeY9/2eQU2nuHeGO/HoChzk8LugpYPQPDvHXoOIVpsSzM9/8+LeM1JT2OgtQYntlea3UU5ee0oKuA9eimKvqGXNwwJyugtvZfLJsIX1s2iZLaDo4c77Y6jvJjWtBVQDrRffooubmORHKTrT0TdCLctjiXyDCbjtLVeQXnpKMKSqM3Eb28t5Ehp5vrZmVamGjiJMdG8IV5Oby8p5H7bpgZtPcL1KXREboKOO29Q+yubWdJYQqpfnQuqK/duTyf3iEXL+8NrGZjauJoQVcBZ3NFKyLCldPSrY4yoRbkJTE7J4Fnt9ditL+LGoMWdBVQegaG2V3bwaL8JBICtJPieIkIdy6fxJHjPezWLoxqDFrQVUDZduwkLrfhM0WhNTo/4+YFOcRHhunNUTUmLegqYAwMu9hedZI5jkTSQmjufLSYiDC+tDiX1/cf5+SpQavjKD+jBV0FjJ3V7Qw63VwRYnPnZ/vasnyGXG7WlTRYHUX5GS3oKiC43IYd1ScpTIvFkRRtdRxLFWXGs6wwhbU7a3G79eao+jMt6CogfFB+go6+YZZPTrU6il/4+mWTqG/v58OKVqujKD+iuxNUQHj6o1oSosKYlR1c7XHH67pZWaTFRfKfbxyhuXPgE8/dsSzfolTKajpCV36vpq2XD4+2sqQgBbsteHu2XIyIMBtrluRRfryHjr4hq+MoP6EFXfm9Z7bXEmYTlhSmWB3Fr9w+MhLfVd1ucRLlL7SgK7826HTx4p4Grp+dRUKQHC3nLY6kaGZkxbOrtgOn2211HOUHtKArv/b2oRY6+4ZZszTP6ih+adnkVHoHnRxs0ra6ysOCLiKrRKRcRCpF5L4xnv9rEdkvIqUiskVEZnk/qgpFL+yqx5EUzYopaVZH8UtTM+JIiY3go2MnrY6i/MAFV7mIiB14GLgWaAB2icgGY8yhUZetNcb878j1q4GfA6t8kFeFkMbOfrZUtvG9q4pC/mbo2nMcEm0TYcWUVP60r5n69j7yUoK/N7w6N09G6EuBSmNMlTFmCHgeuHn0BcaY0e/3YgHd7aAuydoddfxk/QGMgUi77ZwFTcHi/GQiw2xsPdZmdRRlMU8KugOoH/W4YeRznyAi3xGRY8ADwL1jvZCI3CMiJSJS0tqqGyLUubmNYU9tB1PSY0mOjbA6jl+LDLezpCCFA41ddPUPWx1HWciTgj7We91PjcCNMQ8bY6YA/wj881gvZIx51BhTbIwpTk8P7X4c6vyq23rp6Btm8aRkq6MEhMsmp2IMbK/SufRQ5klBbwBGLzHIBZrOc/3zwC2XEkqpsvpOIsJszMpOtDpKQEiOjWBWTgI7q9vpH3JZHUdZxJOCvgsoEpFCEYkA1gAbRl8gIkWjHt4IVHgvogo1A8MuDjR1MTs7gYgwXVnrqZVT0ugfdvHHPdqFMVRd8KfFGOMEvgtsBA4D64wxB0Xk/pEVLQDfFZGDIlIK/D1wl88Sq6D3QfkJBobdzM9LsjpKQJmUGoMjKZont1ZrF8YQ5VFzLmPM68DrZ33uJ6M+/r6Xc6kQ9sreJmIjw5iSHmd1lIAiIqycmsq6kgY2VbRy5fQMqyOpCabvZ5Vf6eof5r3yE8zLTQz5tefjMceRSEZ8JI9vqbY6irKAFnTlVzYeOM6Q082CXJ1uGY8wm427VhSwuaKNoy09VsdRE0wLuvIrr5Q2Mik1htzk0D6V6FLcvjSfyDAbT27VUXqo0QMulOXO7ALt6h/mo2Mn+dyMDER0umW8UmIj+OKiXF7a08A/XD+DFN2YFTJ0hK78xr6GTgzodIsXfGtlAYNON2t31FodRU0gLejKb5TVd+JIiiYtPtLqKAGvKDOeK6al8/RHtQw5tVd6qNCCrvzCie4BmroGWKBrz73mWysLONEzyGv7z7exWwUTLejKL5Q1dCLA3Fzd6u8tVxSlMyU9lse3VGOMbjQKBVrQleWMMZQ1dDElPU6PmfMim0341uWFHGjsZldNh9Vx1ATQVS7KcvXtfbT3DvE53dnoFaN7xw87DdHhdp7YUs1SPWQ76OkIXVmutKGTMJswOyfB6ihBJyLMxtLCFN46dJz69j6r4ygf04KuLDXscrOvoYsZ2QlEhdutjhOUlk9OxSbCU9tqrI6ifEwLurLUloo2+oZcuvbchxKjw7lxXjYv7KqnZ0BPNApmWtCVpdaXNhIdbmdalnZW9KVvrijg1KCT9aW6hDGY6U1RZZm+ISdvHWphjiORMJuOLXzpUFM32YlRPPx+JQIft1a4Y1m+tcGUV+lPkbLM24da6BtyMT9P1577moiwpCCF5q4BGjr6rY6jfEQLurLMK3sbyUmMoiA11uooIWFBXhIRdhs7a9qtjqJ8RAu6skTbqUE2VbSxeoEDm3ZWnBBR4Xbm5yWyr6FTD5IOUlrQlSVe29eMy224ZWGO1VFCytKCVIZdhtJ63TkajLSgK0u8UtrIjKx4ZmTpZqKJ5EiOxpEUzc6adu3vEoS0oKsJV9PWy966Tm5Z6LA6SkhaWpBCS/cgdbpzNOhoQVcTbn1pEyKwer5Ot1hhXl4ikWE2dlbrzdFgowVdTShjDOtLG1lWmEJOkp4baoXIMDsL8pLY39hFV5/uHA0mWtDVhNrX0EVVWy+3LNDpFistLUzB6Tb8cU+D1VGUF+lOUTWhXiltJMJu44a52VZHCWnZidHkJUfzmw+PERlm+8Sh3Lp7NHDpCF1NGKfLzatlzVw1I4PEaD3IwmrFBSm09gzqztEgoiN0NWF+9tph2k4NkhIb8YlDGJQ15joSebWsib31HeSlxFgdR3mBjtDVhCmt7yQq3Mb0rHiroyhO7xydlZNAWX0XTrfb6jjKC7SgqwnRP+TiYHM3c3ISCbfrt52/WJiXTP+wi/LjPVZHUV6gP1lqQmw8eJwhp5sF+XqQhT+ZmhFHfGQYe+s6rY6ivEALupoQ60rqSYmN0M6KfsZuE+bnJVF+vIfeQafVcdQl0oKufK6+vY9tx06yKD9ZOyv6oYX5SbiMYV+DjtIDnRZ05XMv7m5ABBbpdItfyk6MJjsxir31WtADnRZ05VNut+HF3Q1cPjWNpJgIq+Ooc1iYn0xDRz8nugesjqIugRZ05VPbjp2ksbOfrxTnWR1Fncf83ERsgo7SA5wWdOVTf9hdT0JUGNfOyrQ6ijqP+KhwijLiKa3vxOXWPumBSgu68pmuvmHeOHCcWxY6iAq3Wx1HXcDC/CS6+ofZXnXS6ihqnDwq6CKySkTKRaRSRO4b4/m/F5FDIrJPRN4VkUnej6oCzYZ9TQw53Xx5sU63BIKZ2QlEhdv4427twBioLljQRcQOPAzcAMwCbheRWWddthcoNsbMA14EHvB2UBV4/lBSz4yseOY49Ji5QBButzEnJ5E3Dx6nb0jXpAciT0boS4FKY0yVMWYIeB64efQFxpj3jTFnzrPaDuR6N6YKNEeOd7OvoYuvFOd9ojWr8m8L85PpG3Lx9qEWq6OocfCkoDuA+lGPG0Y+dy7fBt4Y6wkRuUdESkSkpLW11fOUKuD89NVD2EVwuQ1rd9Rpd8UAMSk1BkdSNC/vbbQ6ihoHTwr6WMOrMW+Di8idQDHw4FjPG2MeNcYUG2OK09PTPU+pAsqg08Xe+k5mZMcTG6kdmgOJTYSbF+SwuaKN1p5Bq+Ooi+RJQW8ARt/VygWazr5IRK4B/i+w2hij3wkh7M0Dx+kbcrGkIMXqKGocbl3owOU2/Gnfp37MlZ/zpKDvAopEpFBEIoA1wIbRF4jIQuARThfzE96PqQLJs9vrSImNYGpGnNVR1DgUZcYzOyeBV3TaJeBcsKAbY5zAd4GNwGFgnTHmoIjcLyKrRy57EIgD/iAipSKy4Rwvp4Jc+fEedta0s7QgRRtxBbBbFzooa+jiWOspq6Ooi+DROnRjzOvGmGnGmCnGmJ+NfO4nxpgNIx9fY4zJNMYsGPmz+vyvqILVsztqiQizsXhSstVR1CVYPT8Hm6Cj9ACjO0WV1/QOOnlpTyNfmJutN0MDXEZCFCunpvHy3kaM0VYAgUILuvKal/c2cmrQydeW60bhYHDrQgcNHf3sru2wOorykBZ05RVut+HJrdXMdSRq3/MAd2bfQFf/MOF24YGN5VZHUh7Sgq68YlNFK8dae/n25YW6MzRIRIbZmZWdwP6GLoacbqvjKA9oQVde8cTWGjLiI/n83GyroygvWpCXTP+wiw/KdTVyINCCri5ZRUsPm4628o3LJhERpt9SwWRqRhyxkWHaCiBA6E+fumRPbK0hMszG7UvzrY6ivMxuE+bnJvLu4RN09Q9bHUddgBZ0dUlO9Azwxz0NfHGRg9S4SKvjKB9YkJfEkMvNG/ubrY6iLkAXC6tL8uTWGoadbrITo7WjYpByJEUzOT2Wl/c2skbfhfk1HaGrceseGOaZj2qZ7UgkTUfnQUtEuHWBgx3V7TR29lsdR52HFnQ1bs9ur6Nn0Mlnp2kr5GB3y8LTRyCsL9Wbo/5MC7oal4FhF49vqeYzRWk4kqKtjqN8LC8lhiUFyby8R1sB+DMt6GpcnttZR9upQf72yqlWR1ET5IuLcqk4cYrS+k6ro6hz0JuiymNnbnoOOd38/K2jFKbFUt3Wa3EqNVG+MC+b+189xLqSehbmazdNf6QjdHXRdlafpGfQyTUzM62OoiZQfFQ4N87L5tWyZvqGnFbHUWPQgq4uypDTzYcVbUxNj6MwLdbqOGqCfXVJHqcGnby2T9ek+yMt6OqibK86Se+gk6tnZlgdRVmgeFIyk9NjWVdSb3UUNQYt6Mpjg8MuNlW0UpQRx6RUHZ2HIhHhK8V57Krp0OPp/JDeFFVjGmvX50dVJ+kbcunceYj70qJcHtpYznM76vjnL8yyOo4aRQu68sjAsIvNFW1Mz4wnLyXG6jhqgp39C35mdgJ/2N3Aj66fTlS43aJU6mw65aI8su1YG/3DOjpXpy2bnEJX/zCvljVZHUWNogVdXVD/kIstlW3MzE7Akay7QhUUpsZSlBHHM9qQza9oQVcXtKmilYFhN9foyhY1QkT42rJ8yuo72d/QZXUcNUILujqvnoFhth1rY15uItmJOjpXf/bFxblEh9t5Znut1VHUCC3o6rzeL2/F5TZcq3Pn6iwJUeHcstDBK6WNtPcOWR1HoQVdnUdH7xC7qttZPClFTyNSY/rWygIGnW6e26lz6f5Aly2qc3r3SAsicNUMnTtXn3ZmKWNRRhyPfHiM+Kgwwmw27limpxpZRUfoakwt3QPsretk+eRUEqPDrY6j/NiKKal0Dzg50NhtdZSQpwVdjemdwy1EhNn0NCJ1QUWZ8aTFRbDtWJsefmExLejqU8rqOznY1M3lU9OIjdRZOXV+NhFWTEmjoaOfmpN9VscJaVrQ1ac89FY5MRF2Vk5NszqKChCL8pOJjbCz6Wir1VFCmhZ09QnbjrWxuaKNK6ela48O5bGIMBsrpqZR3tLDoSadS7eKFnT1MWMMD20sJyshimWTU62OowLM8sJUIsNs/ObDY1ZHCVla0NXH3j18gj11nXz/miLC7fqtoS5OdISdpYUpvLavidqTetasFfSnVgHgdhseequcgtQYbluca3UcFaBWTk0jzG7jkU1VVkcJSVrQFQCv7mviyPEe/u7aaTo6V+OWEBXObYtzebGkgRPdA1bHCTn6k6sYdrn5+dtHmZEVz03zcqyOowLcX10xGafbzeNbq62OEnI8KugiskpEykWkUkTuG+P5K0Rkj4g4ReQ278dUvrSupJ7ak338w/XTsdnE6jgqwE1KjeXGeTk8u72Orv5hq+OElAsWdBGxAw8DNwCzgNtF5OyDBOuAbwJrvR1Q+dbAsItfvVvB4knJ2rNFec3ffHYKpwad2lp3gnmyDXApUGmMqQIQkeeBm4FDZy4wxtSMPOf2QUblQ7//qJaW7kFump/DczvrrY6jgsCZpl3TMuN4+P1KYiLs3L2y0OJUocGTKRcHMPonvWHkcxdNRO4RkRIRKWlt1R1lVusZGObXH1RSlBHH5LQ4q+OoIHP1jEz6hlx8dOyk1VFChicFfaxJ1XF14DHGPGqMKTbGFKena9Mnqz22uZqOvmGum5VldRQVhPJSYpiRFc+miladS58gnhT0BiBv1ONcQI/6DnAnTw3y2OYqbpiTpQc/K5+5ZmYmA8NuHt+iK14mgicFfRdQJCKFIhIBrAE2+DaW8rXffHCM/mEXP7xumtVRVBDLSYpmjiORJ7ZU6zF1E+CCBd0Y4wS+C2wEDgPrjDEHReR+EVkNICJLRKQB+DLwiIgc9GVodWmau/p5enstX1yUy9SMeKvjqCB39YwMeoecPLJJe7z4mkfNro0xrwOvn/W5n4z6eBenp2JUAPjVuxUYY/jBNUVWR1EhIDMhilsWOPjdthq+fXkhGfFRVkcKWrpTNMT86t0KXthVT/GkFDYdbft4iZlSvvT9q4sYdhl+/b6O0n1JC3qIeedwC3abcOV0XWWkJk5BWixfXpzL2h11NHX2Wx0naGlBDyGHmrrZ19DFyilpxEfpwc9q4qzdUUd+SgwuY7j3ub36ztBHtKCHkIfeKicq3MZninR0riZeUkwEywtT2F3bwfEu7cToC1rQQ0RJTTvvHTnBZ4vSiY7Qo+WUNT43I4OocDuv72/GmHHtT1TnoQU9BBhjeGBjOWlxkVw2RQ9+VtaJiQjj6pkZVLae4r0jJ6yOE3S0oIeATRVt7Kxu596rpxIRpv/LlbWWFaaSFhfJz147zJBT+/l5k/50Bzm32/DgxiPkJkezZkm+1XGUwm4TbpybRVVbL7/drEfVeZMW9CC3vqyRA43d/Oi66To6V35jelYCN8zJ4lfvVuiB0l6kP+FBbGDYxUMbjzLHkcDq+Xq0nPIv/3rTbMLtNv5l/UG9QeolWtCD2FPbamjs7Of/fH6mHi2n/E5WYhQ/um4am462sqFMG7h6g0e9XFTgeWxTFb945yjTM+Opaeujpk03cij/8/XLClhf1sS/vHKAJQUp5CRpK+dLoSP0IPV++QkGh92smqOHVyj/ZbcJ//WVBTjdhh+uK8Pt1qmXS6EFPQjVnuxle1U7xQXJZCZoZzvl3wrSYvnXm2bxUdVJHtuiq14uhRb0IPTAxnLsNuHqmZlWR1HKI18pzuP62Zk88GY5O6r0DNLx0jn0ILO3roPX9jVz1YwMErQBl/JjZzfoWlqQSkXLKf722T1s+N7lOHQ+/aLpCD2IuNyGf91wkPT4SD5TpFv8VWCJjrDz6DeKGXK6uefpEvqHXFZHCjha0IPIM9tr2dfQxb98YRaRYdqASwWendXt3LrIwaGmblb/zxae3lajrXYvghb0INHSPcCDG8v5TFEaN83LtjqOUuM2IyuBWxc6qDhxihdK6nHpyhePaUEPEve/eoghl5uf3jwHEd1EpAJbcUEKn5+bzcGmbv6wu16beHlIC3oQWF/ayGv7m/ne56ZSkBZrdRylvOLyqWlcPyuTfQ1dfPt3uzg16LQ6kt/Tgh7g6tv7+OeXD7B4UjJ/c+UUq+Mo5VWfnZ7BFxc62HbsJGse/Yj69j6rI/k1XbYYwJwuNz94oZQhl5urpmewrqTB6khKeV1xQQo3zc/h3uf2cuOvNvPAbfN1B/Q56Ag9gP2/14+wu7aDmxc4SI6NsDqOUj7zuRkZvHbvZyhIi+Wvn9nND9eV0dozaHUsv6MFPUD9blsNT2yt5u6VBSzIS7I6jlI+l58aw4t/vYK/vXIKG8oaueqhD3h00zF6dW79Y2JVH+Li4mJTUlJiydcOdO8cauGe35dw9cxM/vfOxbywq97qSEpNqLaeQXbXdfDh0VYSo8O567JJfHVpfkjsLhWR3caY4rGe0zn0APPmgWbufa6UOY5EfrlmAXbtc65CUFp8JL/71lJ213bwvx8e41fvVfKr9yopTItlYV4ScxyJRIX/eXPdHctC4/hFLegB5MXdDfz4xTIW5CXx5DeXEhOh//tUaFs8KZnffqOY+vY+Xt7byO+21fDS3kY2lDUxMzuBhXlJFGXGWx1zwmhF8GNntjy73Ia3Dh1nc0UbnylK45GvL9ZirtQoeSkx3Ht1EamxETR09LO3vpN9DZ3sb+wiNsJOQ2cfdy6bRF5KjNVRfUqrgp/r7BvihV311Lb3sawwhcfuKtY+LUrx6W6NACJCXkoMeR8ChFAAAAfqSURBVCkx3Dg3m6MtPeyp6+C3m6p49MMqZmYn8LnpGTiSo4NyGkYLup9yutxsrWzj7cMtAHx1SR7zc5O0mCvlIbtNmJmdwMzsBLr6h9lRfZLtVSc51NzNjKx4lhQkB910jBZ0P2OM4YOjrTz4ZjmHmruZlhnH6vkOUnSduVLjlhgdznWzsriiKJ1tx06ypbKVVb/czF2XFfD9a4pIjA6OswN02aIfWLujDrcxHG3p4cOjrdSe7CM5JpzrZ2cx15GozbaU8rJTg06q23p5flcdKTER/MP10/lycV5ArBrTZYt+rLmrn80Vreysbudk7xAJUWGsnp9DcUEyYTbd96WUL8RFhjHXkUhK7FT+VNbEfS/t57/fq+S/71jIovxkq+ONm47QJ5jLbTjc3M3mijbeLz/Brpp2jIG85GhWTEljjiMxIEYJSgULYwxlDV28eaCZ7gEnXynO5cerZpAWF2l1tDHpCN0iT26tpqVrgKauAY53DdDc1U/bqSH6h08frTUjK54fXD0N4fRGCaXUxBMRFuQlMTMrnvfKT/Di7gY2lDVx7cxMlham8vXLJlkd0WNa0C/SWEulbl+ax/HuAQ43d3OoqZtDzd0cbu6hpq2XM+9/osJtZCdGs2ZpHnMdiVw+NY2MhKhzvqZSamJFhtu5YU42iycl86eyZl7d18yO6nbS4iK4fnYWtgB456xTLhfpqa01tPYM0tI9wPHuAZq6+unoHaKjb/jja/JTYpiVncCw201OYjRZiVEkRYfrzU2lAoQxhoNN3bx1qIW2U4PMzE7g7pUF3DQvh+gIa5cOn2/KxaOCLiKrgF8CduAxY8x/nPV8JPA0sBg4CXzVGFNzvtf0t4L++49q6Rty0jvkom/w9D9n5yTQ0TtEe98Q9e39HG3pob697+NRd5hNyEqM4vKpaczKOb3edUZWPPFRp5dA6chbqcDmNobYSDsPv3+MyhOnSIgK48Z5OVw7K4MVU9I+0S9molzSHLqI2IGHgWuBBmCXiGwwxhwaddm3gQ5jzFQRWQP8J/DVS49+fsYY3Ob0jUaX2+B0u3G6DKcGnfQMOOkZGP7zx4OnH3f2DdPeO/RxoW7vPf2nZ+DcLTjjI8PISoxibm4iRZlxZMZHkZEQSWpsJHabBOWOM6UU2ES4dWEutyxwsKO6nbU76thQ2shzO+uIsNuYkR3PHEcik9NiyU2OJj0+ktjIMGIjwoiLDCMm0k6E3TZh7849mUNfClQaY6oAROR54GZgdEG/Gfi3kY9fBP5HRMT4YD7nsc1VPLixfKSAX/zLR4bZSI2NIDk2gpTYCPKSY0iJjaCho4+YiDBiI8OIibATGxHGncvzSYqJICLsz8sHddStVOgREZZPTmX55FQGnS52VLWztbKN/Y1dvFrWdN4B4em/f/qXg10EEfi31bO5fan3B4IXnHIRkduAVcaYvxh5/HVgmTHmu6OuOTByTcPI42Mj17Sd9Vr3APeMPJwOlHvrX8QL0oC2C17lfwI1NwRu9kDNDYGbPVBzg/ezTzLGpI/1hCcj9LHeK5z9W8CTazDGPAo86sHXnHAiUnKueSl/Fqi5IXCzB2puCNzsgZobJja7J1sRG4C8UY9zgaZzXSMiYUAi0O6NgEoppTzjSUHfBRSJSKGIRABrgA1nXbMBuGvk49uA93wxf66UUurcLjjlYoxxish3gY2cXrb4hDHmoIjcD5QYYzYAjwO/F5FKTo/M1/gytI/45VSQBwI1NwRu9kDNDYGbPVBzwwRmt2xjkVJKKe/Sdn5KKRUktKArpVSQ0II+QkS+LCIHRcQtIgGxPEpEVolIuYhUish9VufxlIg8ISInRvYvBAwRyROR90Xk8Mj3yvetzuQJEYkSkZ0iUjaS+9+tznQxRMQuIntF5E9WZ7kYIlIjIvtFpFREJqTPiRb0PzsAfBHYZHUQT4xqyXADMAu4XURmWZvKY08Bq6wOMQ5O4IfGmJnAcuA7AfLffBC4yhgzH1gArBKR5RZnuhjfBw5bHWKcPmeMWeBP69BDgjHmsDHGn3auXsjHLRmMMUPAmZYMfs8Ys4kA3KdgjGk2xuwZ+biH00XGYW2qCzOnnRp5GD7yJyBWQ4hILnAj8JjVWQKBFvTA5QDqRz1uIACKS7AQkQJgIbDD2iSeGZm2KAVOAG8bYwIiN/AL4MeA2+og42CAt0Rk90jbE58LqQMuROQdIGuMp/6vMWb9ROe5RB61W1DeJyJxwB+BHxhjuq3O4wljjAtYICJJwMsiMscY49f3METkC8AJY8xuEbnS6jzjsNIY0yQiGcDbInJk5N2pz4RUQTfGXGN1Bi/ypCWD8jIRCed0MX/WGPOS1XkuljGmU0Q+4PQ9DL8u6MBKYLWIfB6IAhJE5BljzJ0W5/KIMaZp5J8nRORlTk+T+rSg65RL4PKkJYPyIjnd1Ppx4LAx5udW5/GUiKSPjMwRkWjgGuCItakuzBjzT8aYXGNMAae/v98LlGIuIrEiEn/mY+A6JuAXqBb0ESJyq4g0AJcBr4nIRqsznY8xxgmcaclwGFhnjDlobSrPiMhzwEfAdBFpEJFvW53JQyuBrwNXjSxFKx0ZPfq7bOB9EdnH6YHA28aYgFoCGIAygS0iUgbsBF4zxrzp6y+qW/+VUipI6AhdKaWChBZ0pZQKElrQlVIqSGhBV0qpIKEFXSmlgoQWdKWUChJa0JVSKkj8f/g2AWGOdeHvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(score_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 以下开始特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance = clf_fs.feature_importances_\n",
    "feature_importance = 100.0 * (feature_importance / feature_importance.max())\n",
    "\n",
    "\n",
    "fi_threshold = 18    \n",
    "important_idx = np.where(feature_importance > fi_threshold)[0]\n",
    "important_features = features_list[important_idx]\n",
    "\n",
    "\n",
    "    print(\"\\n\", important_features.shape[0], \"Important features(>\", \\\n",
    "          fi_threshold, \"% of max importance)...\\n\")\n",
    "            #important_features\n",
    "    sorted_idx = np.argsort(feature_importance[important_idx])[::-1]\n",
    "    #get the figure about important features\n",
    "    pos = np.arange(sorted_idx.shape[0]) + .5\n",
    "    plt.subplot(1, 2, 2)\n",
    "    plt.title('Feature Importance')\n",
    "    plt.barh(pos, feature_importance[important_idx][sorted_idx[::-1]], \\\n",
    "            color='r',align='center')\n",
    "    plt.yticks(pos, important_features[sorted_idx[::-1]])\n",
    "    plt.xlabel('Relative Importance')\n",
    "    plt.draw()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., ..., 0., 0., 0.])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_fs_1 = ensemble.GradientBoostingRegressor(random_state=120)\n",
    "clf_fs_1 =clf_fs_1.fit(dg_scaled_train, score_train)\n",
    "clf_fs_1.feature_importances_  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00000000e+00, 3.70501781e-06, 3.50042909e-05, ...,\n",
       "       3.02316896e-04, 1.11858485e-04, 2.12176308e-05])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_fs_2 = RandomForestRegressor(random_state=150)\n",
    "clf_fs_2 =clf_fs_2.fit(dg_scaled_train, score_train)\n",
    "clf_fs_2.feature_importances_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[07:09:44] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.0000000e+00, 1.7383036e-05, 1.4157470e-03, ..., 7.5347588e-04,\n",
       "       0.0000000e+00, 0.0000000e+00], dtype=float32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_fs_3 = xgb.XGBRegressor(random_state=151)\n",
    "clf_fs_3 =clf_fs_3.fit(dg_scaled_train, score_train)\n",
    "clf_fs_3.feature_importances_  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "feature_importance_1=clf_fs_1.feature_importances_\n",
    "feature_importance_2=clf_fs_2.feature_importances_\n",
    "feature_importance_3=clf_fs_3.feature_importances_\n",
    "feature_importance=np.add(feature_importance_1,feature_importance_2,feature_importance_3)\n",
    "#feature_importance\n",
    "index_fs=np.argsort(-feature_importance)\n",
    "index_fs_select=index_fs[0:380]\n",
    "X=dg_scaled_train[:,index_fs_select]\n",
    "X_predict=dg_scaled_test[:,index_fs_select]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  87  919 3111 2613  118 2348 2131 3700 3432 3551  328  699 3115 3214\n",
      " 1360 1335 3371 3465  225  744 1843 2861  192  865 3559 1779 3453 2524\n",
      " 1814  524   14 3318 1258 1618 1689 2418 2631  373 3652 2009 1352  381\n",
      " 2217 1989 2936 3113 1079  607  984 3722 3712 3581 1732 2616 1049  549\n",
      " 2980  720 3289 1473 1978  765  933 3441 3345  418 1412 3129 3262    5\n",
      "  595 2724 1365 1929 3063 1724 3057 2885 3050 3210 2715 2989 1744 3199\n",
      " 1192 2864 3708  875  476  471 3255 3197 1133 2894  119 1795 1262 1189\n",
      " 3650 1314 3759 1197 2377  141  544 1010 3301 2130  670 3733 3174  901\n",
      " 1173 1434 1343  743 2665 2362 3089   57 2267 1973 3070 1384 2988 3659\n",
      " 3761  267 1894 3193 3018 2186  780 2901 2510 2798 1521 2843 2290  776\n",
      " 3238  486  945  280 1161 1065 3669  295 2545 3793 1211 1601 3641  653\n",
      " 3120  256  936  110 3698 2218  442 1948   92 3634  113 2913 2857 2796\n",
      "  529 1872 1930 2833  259 3644 2661 1410 3530 1878 1675   45 3242 3747\n",
      " 1185 1319  458  419 2831  448 3208 3112 2536 2726 1660 1988  217  967\n",
      " 1221 2271 1367 2303  129 3352 3485 2216 1540 3282 2390 1182 1305 1194\n",
      " 2503 2924 1294 2247 2173 1322 2044 2633 3558 2352  247 1363 1138 1722\n",
      " 2522 2762 2024 1548 1012 3447  812 3668 1025 2963   99 1504  316 3042\n",
      "  694 2699  691  209 3660 2499  610 1338 2177 1454 2615  142 1963  279\n",
      "  764 1690 1381 1558  855  353  727 1839 1977 1934 2073 1560 1031 2386\n",
      " 1688  952 1667 2324 1571 3515  872 2820  889 3473 1790 3606  306 3154\n",
      "  768   17 1568 1299 3437 3394 1697 1324 1536 3428 2784  911 1116  881\n",
      " 3395 1260 2520 2907 2723 1391  103 1777 1726 2790 3776 1038 3796 1112\n",
      " 1718 3623 3320 3273 3498  226 1321 1212 3542 2532 1266 3149 2251 2920\n",
      "  659 2746 2065 2773 1467 1537 1816 3003 3016 2091  446 3589 1450 1199\n",
      " 1880 2772 1600  356 2269 1093 1826 1797 1854 2835  747 3482 2714 2873\n",
      "  268 1368  123  236 3774 2823 3767 2457   56 1046 3236 1821 3757 1233\n",
      " 1920 2125  909  566 1863 3096 2704 1325  166 3138  435 2263  898 1749\n",
      "  474 3037]\n"
     ]
    }
   ],
   "source": [
    "print(index_fs_select)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col1</th>\n",
       "      <th>col2</th>\n",
       "      <th>col3</th>\n",
       "      <th>col4</th>\n",
       "      <th>col5</th>\n",
       "      <th>col6</th>\n",
       "      <th>col7</th>\n",
       "      <th>col8</th>\n",
       "      <th>col9</th>\n",
       "      <th>col10</th>\n",
       "      <th>...</th>\n",
       "      <th>col3796</th>\n",
       "      <th>col3797</th>\n",
       "      <th>col3798</th>\n",
       "      <th>col3799</th>\n",
       "      <th>col3800</th>\n",
       "      <th>col3801</th>\n",
       "      <th>col3802</th>\n",
       "      <th>col3803</th>\n",
       "      <th>col3804</th>\n",
       "      <th>col3805</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "      <td>13731.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.000583</td>\n",
       "      <td>4.173750</td>\n",
       "      <td>0.718538</td>\n",
       "      <td>7.142816</td>\n",
       "      <td>12.532379</td>\n",
       "      <td>0.603704</td>\n",
       "      <td>4.345228</td>\n",
       "      <td>0.052560</td>\n",
       "      <td>13.284903</td>\n",
       "      <td>37.756111</td>\n",
       "      <td>...</td>\n",
       "      <td>1.215428</td>\n",
       "      <td>0.106329</td>\n",
       "      <td>2.184837e-07</td>\n",
       "      <td>0.115421</td>\n",
       "      <td>25.719365</td>\n",
       "      <td>19.682047</td>\n",
       "      <td>0.014347</td>\n",
       "      <td>1.879396</td>\n",
       "      <td>0.997325</td>\n",
       "      <td>1.035831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.026981</td>\n",
       "      <td>0.268635</td>\n",
       "      <td>0.118122</td>\n",
       "      <td>26.261926</td>\n",
       "      <td>9.907825</td>\n",
       "      <td>0.169981</td>\n",
       "      <td>0.281740</td>\n",
       "      <td>0.136567</td>\n",
       "      <td>4.676945</td>\n",
       "      <td>7.725197</td>\n",
       "      <td>...</td>\n",
       "      <td>0.443918</td>\n",
       "      <td>0.320774</td>\n",
       "      <td>1.478012e-05</td>\n",
       "      <td>0.033461</td>\n",
       "      <td>28.326643</td>\n",
       "      <td>9.833533</td>\n",
       "      <td>0.118921</td>\n",
       "      <td>1.520451</td>\n",
       "      <td>0.000822</td>\n",
       "      <td>1.755988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.955000</td>\n",
       "      <td>0.386000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.106000</td>\n",
       "      <td>3.457000</td>\n",
       "      <td>-0.399000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>11.672000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.045000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.049000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.039000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.055000</td>\n",
       "      <td>0.991000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.994500</td>\n",
       "      <td>0.628000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.821500</td>\n",
       "      <td>0.463000</td>\n",
       "      <td>4.101000</td>\n",
       "      <td>-0.048000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>32.580000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.093000</td>\n",
       "      <td>1.798500</td>\n",
       "      <td>12.817500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.797000</td>\n",
       "      <td>0.997000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.184000</td>\n",
       "      <td>0.725000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.626000</td>\n",
       "      <td>0.606000</td>\n",
       "      <td>4.344000</td>\n",
       "      <td>0.037000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>37.688000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.181000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.109000</td>\n",
       "      <td>18.147000</td>\n",
       "      <td>17.383000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.651000</td>\n",
       "      <td>0.997000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.374000</td>\n",
       "      <td>0.809000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>18.507500</td>\n",
       "      <td>0.741000</td>\n",
       "      <td>4.582000</td>\n",
       "      <td>0.143000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>42.860500</td>\n",
       "      <td>...</td>\n",
       "      <td>1.476000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.130000</td>\n",
       "      <td>39.595500</td>\n",
       "      <td>24.082000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.763500</td>\n",
       "      <td>0.998000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.893000</td>\n",
       "      <td>0.958000</td>\n",
       "      <td>525.000000</td>\n",
       "      <td>72.823000</td>\n",
       "      <td>0.949000</td>\n",
       "      <td>5.099000</td>\n",
       "      <td>0.561000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>65.224000</td>\n",
       "      <td>...</td>\n",
       "      <td>3.785000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000e-03</td>\n",
       "      <td>0.505000</td>\n",
       "      <td>254.304000</td>\n",
       "      <td>70.825000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>11.219000</td>\n",
       "      <td>0.999000</td>\n",
       "      <td>13.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 3805 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               col1          col2          col3          col4          col5  \\\n",
       "count  13731.000000  13731.000000  13731.000000  13731.000000  13731.000000   \n",
       "mean       0.000583      4.173750      0.718538      7.142816     12.532379   \n",
       "std        0.026981      0.268635      0.118122     26.261926      9.907825   \n",
       "min        0.000000      2.955000      0.386000      0.000000      0.000000   \n",
       "25%        0.000000      3.994500      0.628000      0.000000      4.821500   \n",
       "50%        0.000000      4.184000      0.725000      0.000000     10.626000   \n",
       "75%        0.000000      4.374000      0.809000      4.000000     18.507500   \n",
       "max        2.000000      4.893000      0.958000    525.000000     72.823000   \n",
       "\n",
       "               col6          col7          col8          col9         col10  \\\n",
       "count  13731.000000  13731.000000  13731.000000  13731.000000  13731.000000   \n",
       "mean       0.603704      4.345228      0.052560     13.284903     37.756111   \n",
       "std        0.169981      0.281740      0.136567      4.676945      7.725197   \n",
       "min        0.106000      3.457000     -0.399000      1.000000     11.672000   \n",
       "25%        0.463000      4.101000     -0.048000     10.000000     32.580000   \n",
       "50%        0.606000      4.344000      0.037000     13.000000     37.688000   \n",
       "75%        0.741000      4.582000      0.143000     16.000000     42.860500   \n",
       "max        0.949000      5.099000      0.561000     32.000000     65.224000   \n",
       "\n",
       "       ...       col3796       col3797       col3798       col3799  \\\n",
       "count  ...  13731.000000  13731.000000  1.373100e+04  13731.000000   \n",
       "mean   ...      1.215428      0.106329  2.184837e-07      0.115421   \n",
       "std    ...      0.443918      0.320774  1.478012e-05      0.033461   \n",
       "min    ...      0.045000      0.000000  0.000000e+00      0.049000   \n",
       "25%    ...      0.910000      0.000000  0.000000e+00      0.093000   \n",
       "50%    ...      1.181000      0.000000  0.000000e+00      0.109000   \n",
       "75%    ...      1.476000      0.000000  0.000000e+00      0.130000   \n",
       "max    ...      3.785000      2.000000  1.000000e-03      0.505000   \n",
       "\n",
       "            col3800       col3801       col3802       col3803       col3804  \\\n",
       "count  13731.000000  13731.000000  13731.000000  13731.000000  13731.000000   \n",
       "mean      25.719365     19.682047      0.014347      1.879396      0.997325   \n",
       "std       28.326643      9.833533      0.118921      1.520451      0.000822   \n",
       "min        0.000000      4.039000      0.000000     -2.055000      0.991000   \n",
       "25%        1.798500     12.817500      0.000000      0.797000      0.997000   \n",
       "50%       18.147000     17.383000      0.000000      1.651000      0.997000   \n",
       "75%       39.595500     24.082000      0.000000      2.763500      0.998000   \n",
       "max      254.304000     70.825000      1.000000     11.219000      0.999000   \n",
       "\n",
       "            col3805  \n",
       "count  13731.000000  \n",
       "mean       1.035831  \n",
       "std        1.755988  \n",
       "min        0.000000  \n",
       "25%        0.000000  \n",
       "50%        0.000000  \n",
       "75%        2.000000  \n",
       "max       13.000000  \n",
       "\n",
       "[8 rows x 3805 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dg_train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>3795</th>\n",
       "      <th>3796</th>\n",
       "      <th>3797</th>\n",
       "      <th>3798</th>\n",
       "      <th>3799</th>\n",
       "      <th>3800</th>\n",
       "      <th>3801</th>\n",
       "      <th>3802</th>\n",
       "      <th>3803</th>\n",
       "      <th>3804</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "      <td>1.373100e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.034947e-18</td>\n",
       "      <td>1.029772e-15</td>\n",
       "      <td>-4.398524e-16</td>\n",
       "      <td>-1.448926e-17</td>\n",
       "      <td>-1.557595e-16</td>\n",
       "      <td>-6.571912e-17</td>\n",
       "      <td>3.100701e-15</td>\n",
       "      <td>-1.630041e-17</td>\n",
       "      <td>-2.639114e-17</td>\n",
       "      <td>2.587367e-17</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.529781e-16</td>\n",
       "      <td>1.552420e-17</td>\n",
       "      <td>-8.020838e-18</td>\n",
       "      <td>7.865596e-17</td>\n",
       "      <td>-3.104840e-18</td>\n",
       "      <td>2.654639e-16</td>\n",
       "      <td>-6.209681e-18</td>\n",
       "      <td>9.107532e-17</td>\n",
       "      <td>1.531535e-13</td>\n",
       "      <td>-5.278229e-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "      <td>1.000036e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-2.159435e-02</td>\n",
       "      <td>-4.536983e+00</td>\n",
       "      <td>-2.815307e+00</td>\n",
       "      <td>-2.719936e-01</td>\n",
       "      <td>-1.264943e+00</td>\n",
       "      <td>-2.928107e+00</td>\n",
       "      <td>-3.152769e+00</td>\n",
       "      <td>-3.306627e+00</td>\n",
       "      <td>-2.626790e+00</td>\n",
       "      <td>-3.376621e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.636680e+00</td>\n",
       "      <td>-3.314878e-01</td>\n",
       "      <td>-1.478281e-02</td>\n",
       "      <td>-1.985082e+00</td>\n",
       "      <td>-9.079898e-01</td>\n",
       "      <td>-1.590844e+00</td>\n",
       "      <td>-1.206480e-01</td>\n",
       "      <td>-2.587745e+00</td>\n",
       "      <td>-7.689873e+00</td>\n",
       "      <td>-5.899067e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-2.159435e-02</td>\n",
       "      <td>-6.672854e-01</td>\n",
       "      <td>-7.665088e-01</td>\n",
       "      <td>-2.719936e-01</td>\n",
       "      <td>-7.782899e-01</td>\n",
       "      <td>-8.277924e-01</td>\n",
       "      <td>-8.668872e-01</td>\n",
       "      <td>-7.363688e-01</td>\n",
       "      <td>-7.023864e-01</td>\n",
       "      <td>-6.700540e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.880530e-01</td>\n",
       "      <td>-3.314878e-01</td>\n",
       "      <td>-1.478281e-02</td>\n",
       "      <td>-6.700883e-01</td>\n",
       "      <td>-8.444960e-01</td>\n",
       "      <td>-6.981007e-01</td>\n",
       "      <td>-1.206480e-01</td>\n",
       "      <td>-7.119175e-01</td>\n",
       "      <td>-3.945795e-01</td>\n",
       "      <td>-5.899067e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-2.159435e-02</td>\n",
       "      <td>3.815726e-02</td>\n",
       "      <td>5.470386e-02</td>\n",
       "      <td>-2.719936e-01</td>\n",
       "      <td>-1.924185e-01</td>\n",
       "      <td>1.350988e-02</td>\n",
       "      <td>-4.357328e-03</td>\n",
       "      <td>-1.139417e-01</td>\n",
       "      <td>-6.091865e-02</td>\n",
       "      <td>-8.817025e-03</td>\n",
       "      <td>...</td>\n",
       "      <td>-7.755828e-02</td>\n",
       "      <td>-3.314878e-01</td>\n",
       "      <td>-1.478281e-02</td>\n",
       "      <td>-1.919087e-01</td>\n",
       "      <td>-2.673328e-01</td>\n",
       "      <td>-2.338051e-01</td>\n",
       "      <td>-1.206480e-01</td>\n",
       "      <td>-1.502216e-01</td>\n",
       "      <td>-3.945795e-01</td>\n",
       "      <td>-5.899067e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-2.159435e-02</td>\n",
       "      <td>7.454612e-01</td>\n",
       "      <td>7.658571e-01</td>\n",
       "      <td>-1.196763e-01</td>\n",
       "      <td>6.030928e-01</td>\n",
       "      <td>8.077463e-01</td>\n",
       "      <td>8.404250e-01</td>\n",
       "      <td>6.622616e-01</td>\n",
       "      <td>5.805491e-01</td>\n",
       "      <td>6.607696e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>5.870023e-01</td>\n",
       "      <td>-3.314878e-01</td>\n",
       "      <td>-1.478281e-02</td>\n",
       "      <td>4.357020e-01</td>\n",
       "      <td>4.898794e-01</td>\n",
       "      <td>4.474601e-01</td>\n",
       "      <td>-1.206480e-01</td>\n",
       "      <td>5.814958e-01</td>\n",
       "      <td>8.213028e-01</td>\n",
       "      <td>5.490947e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.410641e+01</td>\n",
       "      <td>2.677518e+00</td>\n",
       "      <td>2.027307e+00</td>\n",
       "      <td>1.971965e+01</td>\n",
       "      <td>6.085374e+00</td>\n",
       "      <td>2.031459e+00</td>\n",
       "      <td>2.675519e+00</td>\n",
       "      <td>3.723139e+00</td>\n",
       "      <td>4.001710e+00</td>\n",
       "      <td>3.555753e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>5.788597e+00</td>\n",
       "      <td>5.903662e+00</td>\n",
       "      <td>6.764614e+01</td>\n",
       "      <td>1.164304e+01</td>\n",
       "      <td>8.069892e+00</td>\n",
       "      <td>5.201062e+00</td>\n",
       "      <td>8.288577e+00</td>\n",
       "      <td>6.142877e+00</td>\n",
       "      <td>2.037185e+00</td>\n",
       "      <td>6.813603e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 3805 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               0             1             2             3             4     \\\n",
       "count  1.373100e+04  1.373100e+04  1.373100e+04  1.373100e+04  1.373100e+04   \n",
       "mean   1.034947e-18  1.029772e-15 -4.398524e-16 -1.448926e-17 -1.557595e-16   \n",
       "std    1.000036e+00  1.000036e+00  1.000036e+00  1.000036e+00  1.000036e+00   \n",
       "min   -2.159435e-02 -4.536983e+00 -2.815307e+00 -2.719936e-01 -1.264943e+00   \n",
       "25%   -2.159435e-02 -6.672854e-01 -7.665088e-01 -2.719936e-01 -7.782899e-01   \n",
       "50%   -2.159435e-02  3.815726e-02  5.470386e-02 -2.719936e-01 -1.924185e-01   \n",
       "75%   -2.159435e-02  7.454612e-01  7.658571e-01 -1.196763e-01  6.030928e-01   \n",
       "max    7.410641e+01  2.677518e+00  2.027307e+00  1.971965e+01  6.085374e+00   \n",
       "\n",
       "               5             6             7             8             9     \\\n",
       "count  1.373100e+04  1.373100e+04  1.373100e+04  1.373100e+04  1.373100e+04   \n",
       "mean  -6.571912e-17  3.100701e-15 -1.630041e-17 -2.639114e-17  2.587367e-17   \n",
       "std    1.000036e+00  1.000036e+00  1.000036e+00  1.000036e+00  1.000036e+00   \n",
       "min   -2.928107e+00 -3.152769e+00 -3.306627e+00 -2.626790e+00 -3.376621e+00   \n",
       "25%   -8.277924e-01 -8.668872e-01 -7.363688e-01 -7.023864e-01 -6.700540e-01   \n",
       "50%    1.350988e-02 -4.357328e-03 -1.139417e-01 -6.091865e-02 -8.817025e-03   \n",
       "75%    8.077463e-01  8.404250e-01  6.622616e-01  5.805491e-01  6.607696e-01   \n",
       "max    2.031459e+00  2.675519e+00  3.723139e+00  4.001710e+00  3.555753e+00   \n",
       "\n",
       "       ...          3795          3796          3797          3798  \\\n",
       "count  ...  1.373100e+04  1.373100e+04  1.373100e+04  1.373100e+04   \n",
       "mean   ... -1.529781e-16  1.552420e-17 -8.020838e-18  7.865596e-17   \n",
       "std    ...  1.000036e+00  1.000036e+00  1.000036e+00  1.000036e+00   \n",
       "min    ... -2.636680e+00 -3.314878e-01 -1.478281e-02 -1.985082e+00   \n",
       "25%    ... -6.880530e-01 -3.314878e-01 -1.478281e-02 -6.700883e-01   \n",
       "50%    ... -7.755828e-02 -3.314878e-01 -1.478281e-02 -1.919087e-01   \n",
       "75%    ...  5.870023e-01 -3.314878e-01 -1.478281e-02  4.357020e-01   \n",
       "max    ...  5.788597e+00  5.903662e+00  6.764614e+01  1.164304e+01   \n",
       "\n",
       "               3799          3800          3801          3802          3803  \\\n",
       "count  1.373100e+04  1.373100e+04  1.373100e+04  1.373100e+04  1.373100e+04   \n",
       "mean  -3.104840e-18  2.654639e-16 -6.209681e-18  9.107532e-17  1.531535e-13   \n",
       "std    1.000036e+00  1.000036e+00  1.000036e+00  1.000036e+00  1.000036e+00   \n",
       "min   -9.079898e-01 -1.590844e+00 -1.206480e-01 -2.587745e+00 -7.689873e+00   \n",
       "25%   -8.444960e-01 -6.981007e-01 -1.206480e-01 -7.119175e-01 -3.945795e-01   \n",
       "50%   -2.673328e-01 -2.338051e-01 -1.206480e-01 -1.502216e-01 -3.945795e-01   \n",
       "75%    4.898794e-01  4.474601e-01 -1.206480e-01  5.814958e-01  8.213028e-01   \n",
       "max    8.069892e+00  5.201062e+00  8.288577e+00  6.142877e+00  2.037185e+00   \n",
       "\n",
       "               3804  \n",
       "count  1.373100e+04  \n",
       "mean  -5.278229e-17  \n",
       "std    1.000036e+00  \n",
       "min   -5.899067e-01  \n",
       "25%   -5.899067e-01  \n",
       "50%   -5.899067e-01  \n",
       "75%    5.490947e-01  \n",
       "max    6.813603e+00  \n",
       "\n",
       "[8 rows x 3805 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(dg_scaled_train).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    13731.000000\n",
       "mean        34.812465\n",
       "std         34.459633\n",
       "min          0.010000\n",
       "25%          1.900000\n",
       "50%         24.000000\n",
       "75%         63.000000\n",
       "max        230.000000\n",
       "Name: col2614, dtype: float64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dg_train[\"col2614\"].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "289\n",
      "290\n",
      "291\n",
      "292\n",
      "293\n",
      "294\n",
      "295\n",
      "296\n",
      "297\n",
      "298\n",
      "299\n",
      "300\n",
      "301\n",
      "302\n",
      "303\n",
      "304\n",
      "305\n",
      "306\n",
      "307\n",
      "308\n",
      "309\n",
      "310\n",
      "311\n",
      "312\n",
      "313\n",
      "314\n",
      "315\n",
      "316\n",
      "317\n",
      "318\n",
      "319\n",
      "320\n",
      "321\n",
      "322\n",
      "323\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "332\n",
      "333\n",
      "334\n",
      "335\n",
      "336\n",
      "337\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "343\n",
      "344\n",
      "345\n",
      "346\n",
      "347\n",
      "348\n",
      "349\n",
      "350\n",
      "351\n",
      "352\n",
      "353\n",
      "354\n",
      "355\n",
      "356\n",
      "357\n",
      "358\n",
      "359\n",
      "360\n",
      "361\n",
      "362\n",
      "363\n",
      "364\n",
      "365\n",
      "366\n",
      "367\n",
      "368\n",
      "369\n",
      "370\n",
      "371\n",
      "372\n",
      "373\n",
      "374\n",
      "375\n",
      "376\n",
      "377\n",
      "378\n",
      "379\n",
      "380\n",
      "381\n",
      "382\n",
      "383\n",
      "384\n",
      "385\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "391\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n",
      "400\n",
      "401\n",
      "402\n",
      "403\n",
      "404\n",
      "405\n",
      "406\n",
      "407\n",
      "408\n",
      "409\n",
      "410\n",
      "411\n",
      "412\n",
      "413\n",
      "414\n",
      "415\n",
      "416\n",
      "417\n",
      "418\n",
      "419\n",
      "420\n",
      "421\n",
      "422\n",
      "423\n",
      "424\n",
      "425\n",
      "426\n",
      "427\n",
      "428\n",
      "429\n",
      "430\n",
      "431\n",
      "432\n",
      "433\n",
      "434\n",
      "435\n",
      "436\n",
      "437\n",
      "438\n",
      "439\n",
      "440\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n",
      "447\n",
      "448\n",
      "449\n",
      "450\n",
      "451\n",
      "452\n",
      "453\n",
      "454\n",
      "455\n",
      "456\n",
      "457\n",
      "458\n",
      "459\n",
      "460\n",
      "461\n",
      "462\n",
      "463\n",
      "464\n",
      "465\n",
      "466\n",
      "467\n",
      "468\n",
      "469\n",
      "470\n",
      "471\n",
      "472\n",
      "473\n",
      "474\n",
      "475\n",
      "476\n",
      "477\n",
      "478\n",
      "479\n",
      "480\n",
      "481\n",
      "482\n",
      "483\n",
      "484\n",
      "485\n",
      "486\n",
      "487\n",
      "488\n",
      "489\n",
      "490\n",
      "491\n",
      "492\n",
      "493\n",
      "494\n",
      "495\n",
      "496\n",
      "497\n",
      "498\n",
      "499\n",
      "500\n",
      "501\n",
      "502\n",
      "503\n",
      "504\n",
      "505\n",
      "506\n",
      "507\n",
      "508\n",
      "509\n",
      "510\n",
      "511\n",
      "512\n",
      "513\n",
      "514\n",
      "515\n",
      "516\n",
      "517\n",
      "518\n",
      "519\n",
      "520\n",
      "521\n",
      "522\n",
      "523\n",
      "524\n",
      "525\n",
      "526\n",
      "527\n",
      "528\n",
      "529\n",
      "530\n",
      "531\n",
      "532\n",
      "533\n",
      "534\n",
      "535\n",
      "536\n",
      "537\n",
      "538\n",
      "539\n",
      "540\n",
      "541\n",
      "542\n",
      "543\n",
      "544\n",
      "545\n",
      "546\n",
      "547\n",
      "548\n",
      "549\n",
      "550\n",
      "551\n",
      "552\n",
      "553\n",
      "554\n",
      "555\n",
      "556\n",
      "557\n",
      "558\n",
      "559\n",
      "560\n",
      "561\n",
      "562\n",
      "563\n",
      "564\n",
      "565\n",
      "566\n",
      "567\n",
      "568\n",
      "569\n",
      "570\n",
      "571\n",
      "572\n",
      "573\n",
      "574\n",
      "575\n",
      "576\n",
      "577\n",
      "578\n",
      "579\n",
      "580\n",
      "581\n",
      "582\n",
      "583\n",
      "584\n",
      "585\n",
      "586\n",
      "587\n",
      "588\n",
      "589\n",
      "590\n",
      "591\n",
      "592\n",
      "593\n",
      "594\n",
      "595\n",
      "596\n",
      "597\n",
      "598\n",
      "599\n",
      "600\n",
      "601\n",
      "602\n",
      "603\n",
      "604\n",
      "605\n",
      "606\n",
      "607\n",
      "608\n",
      "609\n",
      "610\n",
      "611\n",
      "612\n",
      "613\n",
      "614\n",
      "615\n",
      "616\n",
      "617\n",
      "618\n",
      "619\n",
      "620\n",
      "621\n",
      "622\n",
      "623\n",
      "624\n",
      "625\n",
      "626\n",
      "627\n",
      "628\n",
      "629\n",
      "630\n",
      "631\n",
      "632\n",
      "633\n",
      "634\n",
      "635\n",
      "636\n",
      "637\n",
      "638\n",
      "639\n",
      "640\n",
      "641\n",
      "642\n",
      "643\n",
      "644\n",
      "645\n",
      "646\n",
      "647\n",
      "648\n",
      "649\n",
      "650\n",
      "651\n",
      "652\n",
      "653\n",
      "654\n",
      "655\n",
      "656\n",
      "657\n",
      "658\n",
      "659\n",
      "660\n",
      "661\n",
      "662\n",
      "663\n",
      "664\n",
      "665\n",
      "666\n",
      "667\n",
      "668\n",
      "669\n",
      "670\n",
      "671\n",
      "672\n",
      "673\n",
      "674\n",
      "675\n",
      "676\n",
      "677\n",
      "678\n",
      "679\n",
      "680\n",
      "681\n",
      "682\n",
      "683\n",
      "684\n",
      "685\n",
      "686\n",
      "687\n",
      "688\n",
      "689\n",
      "690\n",
      "691\n",
      "692\n",
      "693\n",
      "694\n",
      "695\n",
      "696\n",
      "697\n",
      "698\n",
      "699\n",
      "700\n",
      "701\n",
      "702\n",
      "703\n",
      "704\n",
      "705\n",
      "706\n",
      "707\n",
      "708\n",
      "709\n",
      "710\n",
      "711\n",
      "712\n",
      "713\n",
      "714\n",
      "715\n",
      "716\n",
      "717\n",
      "718\n",
      "719\n",
      "720\n",
      "721\n",
      "722\n",
      "723\n",
      "724\n",
      "725\n",
      "726\n",
      "727\n",
      "728\n",
      "729\n",
      "730\n",
      "731\n",
      "732\n",
      "733\n",
      "734\n",
      "735\n",
      "736\n",
      "737\n",
      "738\n",
      "739\n",
      "740\n",
      "741\n",
      "742\n",
      "743\n",
      "744\n",
      "745\n",
      "746\n",
      "747\n",
      "748\n",
      "749\n",
      "750\n",
      "751\n",
      "752\n",
      "753\n",
      "754\n",
      "755\n",
      "756\n",
      "757\n",
      "758\n",
      "759\n",
      "760\n",
      "761\n",
      "762\n",
      "763\n",
      "764\n",
      "765\n",
      "766\n",
      "767\n",
      "768\n",
      "769\n",
      "770\n",
      "771\n",
      "772\n",
      "773\n",
      "774\n",
      "775\n",
      "776\n",
      "777\n",
      "778\n",
      "779\n",
      "780\n",
      "781\n",
      "782\n",
      "783\n",
      "784\n",
      "785\n",
      "786\n",
      "787\n",
      "788\n",
      "789\n",
      "790\n",
      "791\n",
      "792\n",
      "793\n",
      "794\n",
      "795\n",
      "796\n",
      "797\n",
      "798\n",
      "799\n",
      "800\n",
      "801\n",
      "802\n",
      "803\n",
      "804\n",
      "805\n",
      "806\n",
      "807\n",
      "808\n",
      "809\n",
      "810\n",
      "811\n",
      "812\n",
      "813\n",
      "814\n",
      "815\n",
      "816\n",
      "817\n",
      "818\n",
      "819\n",
      "820\n",
      "821\n",
      "822\n",
      "823\n",
      "824\n",
      "825\n",
      "826\n",
      "827\n",
      "828\n",
      "829\n",
      "830\n",
      "831\n",
      "832\n",
      "833\n",
      "834\n",
      "835\n",
      "836\n",
      "837\n",
      "838\n",
      "839\n",
      "840\n",
      "841\n",
      "842\n",
      "843\n",
      "844\n",
      "845\n",
      "846\n",
      "847\n",
      "848\n",
      "849\n",
      "850\n",
      "851\n",
      "852\n",
      "853\n",
      "854\n",
      "855\n",
      "856\n",
      "857\n",
      "858\n",
      "859\n",
      "860\n",
      "861\n",
      "862\n",
      "863\n",
      "864\n",
      "865\n",
      "866\n",
      "867\n",
      "868\n",
      "869\n",
      "870\n",
      "871\n",
      "872\n",
      "873\n",
      "874\n",
      "875\n",
      "876\n",
      "877\n",
      "878\n",
      "879\n",
      "880\n",
      "881\n",
      "882\n",
      "883\n",
      "884\n",
      "885\n",
      "886\n",
      "887\n",
      "888\n",
      "889\n",
      "890\n",
      "891\n",
      "892\n",
      "893\n",
      "894\n",
      "895\n",
      "896\n",
      "897\n",
      "898\n",
      "899\n",
      "900\n",
      "901\n",
      "902\n",
      "903\n",
      "904\n",
      "905\n",
      "906\n",
      "907\n",
      "908\n",
      "909\n",
      "910\n",
      "911\n",
      "912\n",
      "913\n",
      "914\n",
      "915\n",
      "916\n",
      "917\n",
      "918\n",
      "919\n",
      "920\n",
      "921\n",
      "922\n",
      "923\n",
      "924\n",
      "925\n",
      "926\n",
      "927\n",
      "928\n",
      "929\n",
      "930\n",
      "931\n",
      "932\n",
      "933\n",
      "934\n",
      "935\n",
      "936\n",
      "937\n",
      "938\n",
      "939\n",
      "940\n",
      "941\n",
      "942\n",
      "943\n",
      "944\n",
      "945\n",
      "946\n",
      "947\n",
      "948\n",
      "949\n",
      "950\n",
      "951\n",
      "952\n",
      "953\n",
      "954\n",
      "955\n",
      "956\n",
      "957\n",
      "958\n",
      "959\n",
      "960\n",
      "961\n",
      "962\n",
      "963\n",
      "964\n",
      "965\n",
      "966\n",
      "967\n",
      "968\n",
      "969\n",
      "970\n",
      "971\n",
      "972\n",
      "973\n",
      "974\n",
      "975\n",
      "976\n",
      "977\n",
      "978\n",
      "979\n",
      "980\n",
      "981\n",
      "982\n",
      "983\n",
      "984\n",
      "985\n",
      "986\n",
      "987\n",
      "988\n",
      "989\n",
      "990\n",
      "991\n",
      "992\n",
      "993\n",
      "994\n",
      "995\n",
      "996\n",
      "997\n",
      "998\n",
      "999\n",
      "1000\n",
      "1001\n",
      "1002\n",
      "1003\n",
      "1004\n",
      "1005\n",
      "1006\n",
      "1007\n",
      "1008\n",
      "1009\n",
      "1010\n",
      "1011\n",
      "1012\n",
      "1013\n",
      "1014\n",
      "1015\n",
      "1016\n",
      "1017\n",
      "1018\n",
      "1019\n",
      "1020\n",
      "1021\n",
      "1022\n",
      "1023\n",
      "1024\n",
      "1025\n",
      "1026\n",
      "1027\n",
      "1028\n",
      "1029\n",
      "1030\n",
      "1031\n",
      "1032\n",
      "1033\n",
      "1034\n",
      "1035\n",
      "1036\n",
      "1037\n",
      "1038\n",
      "1039\n",
      "1040\n",
      "1041\n",
      "1042\n",
      "1043\n",
      "1044\n",
      "1045\n",
      "1046\n",
      "1047\n",
      "1048\n",
      "1049\n",
      "1050\n",
      "1051\n",
      "1052\n",
      "1053\n",
      "1054\n",
      "1055\n",
      "1056\n",
      "1057\n",
      "1058\n",
      "1059\n",
      "1060\n",
      "1061\n",
      "1062\n",
      "1063\n",
      "1064\n",
      "1065\n",
      "1066\n",
      "1067\n",
      "1068\n",
      "1069\n",
      "1070\n",
      "1071\n",
      "1072\n",
      "1073\n",
      "1074\n",
      "1075\n",
      "1076\n",
      "1077\n",
      "1078\n",
      "1079\n",
      "1080\n",
      "1081\n",
      "1082\n",
      "1083\n",
      "1084\n",
      "1085\n",
      "1086\n",
      "1087\n",
      "1088\n",
      "1089\n",
      "1090\n",
      "1091\n",
      "1092\n",
      "1093\n",
      "1094\n",
      "1095\n",
      "1096\n",
      "1097\n",
      "1098\n",
      "1099\n",
      "1100\n",
      "1101\n",
      "1102\n",
      "1103\n",
      "1104\n",
      "1105\n",
      "1106\n",
      "1107\n",
      "1108\n",
      "1109\n",
      "1110\n",
      "1111\n",
      "1112\n",
      "1113\n",
      "1114\n",
      "1115\n",
      "1116\n",
      "1117\n",
      "1118\n",
      "1119\n",
      "1120\n",
      "1121\n",
      "1122\n",
      "1123\n",
      "1124\n",
      "1125\n",
      "1126\n",
      "1127\n",
      "1128\n",
      "1129\n",
      "1130\n",
      "1131\n",
      "1132\n",
      "1133\n",
      "1134\n",
      "1135\n",
      "1136\n",
      "1137\n",
      "1138\n",
      "1139\n",
      "1140\n",
      "1141\n",
      "1142\n",
      "1143\n",
      "1144\n",
      "1145\n",
      "1146\n",
      "1147\n",
      "1148\n",
      "1149\n",
      "1150\n",
      "1151\n",
      "1152\n",
      "1153\n",
      "1154\n",
      "1155\n",
      "1156\n",
      "1157\n",
      "1158\n",
      "1159\n",
      "1160\n",
      "1161\n",
      "1162\n",
      "1163\n",
      "1164\n",
      "1165\n",
      "1166\n",
      "1167\n",
      "1168\n",
      "1169\n",
      "1170\n",
      "1171\n",
      "1172\n",
      "1173\n",
      "1174\n",
      "1175\n",
      "1176\n",
      "1177\n",
      "1178\n",
      "1179\n",
      "1180\n",
      "1181\n",
      "1182\n",
      "1183\n",
      "1184\n",
      "1185\n",
      "1186\n",
      "1187\n",
      "1188\n",
      "1189\n",
      "1190\n",
      "1191\n",
      "1192\n",
      "1193\n",
      "1194\n",
      "1195\n",
      "1196\n",
      "1197\n",
      "1198\n",
      "1199\n",
      "1200\n",
      "1201\n",
      "1202\n",
      "1203\n",
      "1204\n",
      "1205\n",
      "1206\n",
      "1207\n",
      "1208\n",
      "1209\n",
      "1210\n",
      "1211\n",
      "1212\n",
      "1213\n",
      "1214\n",
      "1215\n",
      "1216\n",
      "1217\n",
      "1218\n",
      "1219\n",
      "1220\n",
      "1221\n",
      "1222\n",
      "1223\n",
      "1224\n",
      "1225\n",
      "1226\n",
      "1227\n",
      "1228\n",
      "1229\n",
      "1230\n",
      "1231\n",
      "1232\n",
      "1233\n",
      "1234\n",
      "1235\n",
      "1236\n",
      "1237\n",
      "1238\n",
      "1239\n",
      "1240\n",
      "1241\n",
      "1242\n",
      "1243\n",
      "1244\n",
      "1245\n",
      "1246\n",
      "1247\n",
      "1248\n",
      "1249\n",
      "1250\n",
      "1251\n",
      "1252\n",
      "1253\n",
      "1254\n",
      "1255\n",
      "1256\n",
      "1257\n",
      "1258\n",
      "1259\n",
      "1260\n",
      "1261\n",
      "1262\n",
      "1263\n",
      "1264\n",
      "1265\n",
      "1266\n",
      "1267\n",
      "1268\n",
      "1269\n",
      "1270\n",
      "1271\n",
      "1272\n",
      "1273\n",
      "1274\n",
      "1275\n",
      "1276\n",
      "1277\n",
      "1278\n",
      "1279\n",
      "1280\n",
      "1281\n",
      "1282\n",
      "1283\n",
      "1284\n",
      "1285\n",
      "1286\n",
      "1287\n",
      "1288\n",
      "1289\n",
      "1290\n",
      "1291\n",
      "1292\n",
      "1293\n",
      "1294\n",
      "1295\n",
      "1296\n",
      "1297\n",
      "1298\n",
      "1299\n",
      "1300\n",
      "1301\n",
      "1302\n",
      "1303\n",
      "1304\n",
      "1305\n",
      "1306\n",
      "1307\n",
      "1308\n",
      "1309\n",
      "1310\n",
      "1311\n",
      "1312\n",
      "1313\n",
      "1314\n",
      "1315\n",
      "1316\n",
      "1317\n",
      "1318\n",
      "1319\n",
      "1320\n",
      "1321\n",
      "1322\n",
      "1323\n",
      "1324\n",
      "1325\n",
      "1326\n",
      "1327\n",
      "1328\n",
      "1329\n",
      "1330\n",
      "1331\n",
      "1332\n",
      "1333\n",
      "1334\n",
      "1335\n",
      "1336\n",
      "1337\n",
      "1338\n",
      "1339\n",
      "1340\n",
      "1341\n",
      "1342\n",
      "1343\n",
      "1344\n",
      "1345\n",
      "1346\n",
      "1347\n",
      "1348\n",
      "1349\n",
      "1350\n",
      "1351\n",
      "1352\n",
      "1353\n",
      "1354\n",
      "1355\n",
      "1356\n",
      "1357\n",
      "1358\n",
      "1359\n",
      "1360\n",
      "1361\n",
      "1362\n",
      "1363\n",
      "1364\n",
      "1365\n",
      "1366\n",
      "1367\n",
      "1368\n",
      "1369\n",
      "1370\n",
      "1371\n",
      "1372\n",
      "1373\n",
      "1374\n",
      "1375\n",
      "1376\n",
      "1377\n",
      "1378\n",
      "1379\n",
      "1380\n",
      "1381\n",
      "1382\n",
      "1383\n",
      "1384\n",
      "1385\n",
      "1386\n",
      "1387\n",
      "1388\n",
      "1389\n",
      "1390\n",
      "1391\n",
      "1392\n",
      "1393\n",
      "1394\n",
      "1395\n",
      "1396\n",
      "1397\n",
      "1398\n",
      "1399\n",
      "1400\n",
      "1401\n",
      "1402\n",
      "1403\n",
      "1404\n",
      "1405\n",
      "1406\n",
      "1407\n",
      "1408\n",
      "1409\n",
      "1410\n",
      "1411\n",
      "1412\n",
      "1413\n",
      "1414\n",
      "1415\n",
      "1416\n",
      "1417\n",
      "1418\n",
      "1419\n",
      "1420\n",
      "1421\n",
      "1422\n",
      "1423\n",
      "1424\n",
      "1425\n",
      "1426\n",
      "1427\n",
      "1428\n",
      "1429\n",
      "1430\n",
      "1431\n",
      "1432\n",
      "1433\n",
      "1434\n",
      "1435\n",
      "1436\n",
      "1437\n",
      "1438\n",
      "1439\n",
      "1440\n",
      "1441\n",
      "1442\n",
      "1443\n",
      "1444\n",
      "1445\n",
      "1446\n",
      "1447\n",
      "1448\n",
      "1449\n",
      "1450\n",
      "1451\n",
      "1452\n",
      "1453\n",
      "1454\n",
      "1455\n",
      "1456\n",
      "1457\n",
      "1458\n",
      "1459\n",
      "1460\n",
      "1461\n",
      "1462\n",
      "1463\n",
      "1464\n",
      "1465\n",
      "1466\n",
      "1467\n",
      "1468\n",
      "1469\n",
      "1470\n",
      "1471\n",
      "1472\n",
      "1473\n",
      "1474\n",
      "1475\n",
      "1476\n",
      "1477\n",
      "1478\n",
      "1479\n",
      "1480\n",
      "1481\n",
      "1482\n",
      "1483\n",
      "1484\n",
      "1485\n",
      "1486\n",
      "1487\n",
      "1488\n",
      "1489\n",
      "1490\n",
      "1491\n",
      "1492\n",
      "1493\n",
      "1494\n",
      "1495\n",
      "1496\n",
      "1497\n",
      "1498\n",
      "1499\n",
      "1500\n",
      "1501\n",
      "1502\n",
      "1503\n",
      "1504\n",
      "1505\n",
      "1506\n",
      "1507\n",
      "1508\n",
      "1509\n",
      "1510\n",
      "1511\n",
      "1512\n",
      "1513\n",
      "1514\n",
      "1515\n",
      "1516\n",
      "1517\n",
      "1518\n",
      "1519\n",
      "1520\n",
      "1521\n",
      "1522\n",
      "1523\n",
      "1524\n",
      "1525\n",
      "1526\n",
      "1527\n",
      "1528\n",
      "1529\n",
      "1530\n",
      "1531\n",
      "1532\n",
      "1533\n",
      "1534\n",
      "1535\n",
      "1536\n",
      "1537\n",
      "1538\n",
      "1539\n",
      "1540\n",
      "1541\n",
      "1542\n",
      "1543\n",
      "1544\n",
      "1545\n",
      "1546\n",
      "1547\n",
      "1548\n",
      "1549\n",
      "1550\n",
      "1551\n",
      "1552\n",
      "1553\n",
      "1554\n",
      "1555\n",
      "1556\n",
      "1557\n",
      "1558\n",
      "1559\n",
      "1560\n",
      "1561\n",
      "1562\n",
      "1563\n",
      "1564\n",
      "1565\n",
      "1566\n",
      "1567\n",
      "1568\n",
      "1569\n",
      "1570\n",
      "1571\n",
      "1572\n",
      "1573\n",
      "1574\n",
      "1575\n",
      "1576\n",
      "1577\n",
      "1578\n",
      "1579\n",
      "1580\n",
      "1581\n",
      "1582\n",
      "1583\n",
      "1584\n",
      "1585\n",
      "1586\n",
      "1587\n",
      "1588\n",
      "1589\n",
      "1590\n",
      "1591\n",
      "1592\n",
      "1593\n",
      "1594\n",
      "1595\n",
      "1596\n",
      "1597\n",
      "1598\n",
      "1599\n",
      "1600\n",
      "1601\n",
      "1602\n",
      "1603\n",
      "1604\n",
      "1605\n",
      "1606\n",
      "1607\n",
      "1608\n",
      "1609\n",
      "1610\n",
      "1611\n",
      "1612\n",
      "1613\n",
      "1614\n",
      "1615\n",
      "1616\n",
      "1617\n",
      "1618\n",
      "1619\n",
      "1620\n",
      "1621\n",
      "1622\n",
      "1623\n",
      "1624\n",
      "1625\n",
      "1626\n",
      "1627\n",
      "1628\n",
      "1629\n",
      "1630\n",
      "1631\n",
      "1632\n",
      "1633\n",
      "1634\n",
      "1635\n",
      "1636\n",
      "1637\n",
      "1638\n",
      "1639\n",
      "1640\n",
      "1641\n",
      "1642\n",
      "1643\n",
      "1644\n",
      "1645\n",
      "1646\n",
      "1647\n",
      "1648\n",
      "1649\n",
      "1650\n",
      "1651\n",
      "1652\n",
      "1653\n",
      "1654\n",
      "1655\n",
      "1656\n",
      "1657\n",
      "1658\n",
      "1659\n",
      "1660\n",
      "1661\n",
      "1662\n",
      "1663\n",
      "1664\n",
      "1665\n",
      "1666\n",
      "1667\n",
      "1668\n",
      "1669\n",
      "1670\n",
      "1671\n",
      "1672\n",
      "1673\n",
      "1674\n",
      "1675\n",
      "1676\n",
      "1677\n",
      "1678\n",
      "1679\n",
      "1680\n",
      "1681\n",
      "1682\n",
      "1683\n",
      "1684\n",
      "1685\n",
      "1686\n",
      "1687\n",
      "1688\n",
      "1689\n",
      "1690\n",
      "1691\n",
      "1692\n",
      "1693\n",
      "1694\n",
      "1695\n",
      "1696\n",
      "1697\n",
      "1698\n",
      "1699\n",
      "1700\n",
      "1701\n",
      "1702\n",
      "1703\n",
      "1704\n",
      "1705\n",
      "1706\n",
      "1707\n",
      "1708\n",
      "1709\n",
      "1710\n",
      "1711\n",
      "1712\n",
      "1713\n",
      "1714\n",
      "1715\n",
      "1716\n",
      "1717\n",
      "1718\n",
      "1719\n",
      "1720\n",
      "1721\n",
      "1722\n",
      "1723\n",
      "1724\n",
      "1725\n",
      "1726\n",
      "1727\n",
      "1728\n",
      "1729\n",
      "1730\n",
      "1731\n",
      "1732\n",
      "1733\n",
      "1734\n",
      "1735\n",
      "1736\n",
      "1737\n",
      "1738\n",
      "1739\n",
      "1740\n",
      "1741\n",
      "1742\n",
      "1743\n",
      "1744\n",
      "1745\n",
      "1746\n",
      "1747\n",
      "1748\n",
      "1749\n",
      "1750\n",
      "1751\n",
      "1752\n",
      "1753\n",
      "1754\n",
      "1755\n",
      "1756\n",
      "1757\n",
      "1758\n",
      "1759\n",
      "1760\n",
      "1761\n",
      "1762\n",
      "1763\n",
      "1764\n",
      "1765\n",
      "1766\n",
      "1767\n",
      "1768\n",
      "1769\n",
      "1770\n",
      "1771\n",
      "1772\n",
      "1773\n",
      "1774\n",
      "1775\n",
      "1776\n",
      "1777\n",
      "1778\n",
      "1779\n",
      "1780\n",
      "1781\n",
      "1782\n",
      "1783\n",
      "1784\n",
      "1785\n",
      "1786\n",
      "1787\n",
      "1788\n",
      "1789\n",
      "1790\n",
      "1791\n",
      "1792\n",
      "1793\n",
      "1794\n",
      "1795\n",
      "1796\n",
      "1797\n",
      "1798\n",
      "1799\n",
      "1800\n",
      "1801\n",
      "1802\n",
      "1803\n",
      "1804\n",
      "1805\n",
      "1806\n",
      "1807\n",
      "1808\n",
      "1809\n",
      "1810\n",
      "1811\n",
      "1812\n",
      "1813\n",
      "1814\n",
      "1815\n",
      "1816\n",
      "1817\n",
      "1818\n",
      "1819\n",
      "1820\n",
      "1821\n",
      "1822\n",
      "1823\n",
      "1824\n",
      "1825\n",
      "1826\n",
      "1827\n",
      "1828\n",
      "1829\n",
      "1830\n",
      "1831\n",
      "1832\n",
      "1833\n",
      "1834\n",
      "1835\n",
      "1836\n",
      "1837\n",
      "1838\n",
      "1839\n",
      "1840\n",
      "1841\n",
      "1842\n",
      "1843\n",
      "1844\n",
      "1845\n",
      "1846\n",
      "1847\n",
      "1848\n",
      "1849\n",
      "1850\n",
      "1851\n",
      "1852\n",
      "1853\n",
      "1854\n",
      "1855\n",
      "1856\n",
      "1857\n",
      "1858\n",
      "1859\n",
      "1860\n",
      "1861\n",
      "1862\n",
      "1863\n",
      "1864\n",
      "1865\n",
      "1866\n",
      "1867\n",
      "1868\n",
      "1869\n",
      "1870\n",
      "1871\n",
      "1872\n",
      "1873\n",
      "1874\n",
      "1875\n",
      "1876\n",
      "1877\n",
      "1878\n",
      "1879\n",
      "1880\n",
      "1881\n",
      "1882\n",
      "1883\n",
      "1884\n",
      "1885\n",
      "1886\n",
      "1887\n",
      "1888\n",
      "1889\n",
      "1890\n",
      "1891\n",
      "1892\n",
      "1893\n",
      "1894\n",
      "1895\n",
      "1896\n",
      "1897\n",
      "1898\n",
      "1899\n",
      "1900\n",
      "1901\n",
      "1902\n",
      "1903\n",
      "1904\n",
      "1905\n",
      "1906\n",
      "1907\n",
      "1908\n",
      "1909\n",
      "1910\n",
      "1911\n",
      "1912\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1913\n",
      "1914\n",
      "1915\n",
      "1916\n",
      "1917\n",
      "1918\n",
      "1919\n",
      "1920\n",
      "1921\n",
      "1922\n",
      "1923\n",
      "1924\n",
      "1925\n",
      "1926\n",
      "1927\n",
      "1928\n",
      "1929\n",
      "1930\n",
      "1931\n",
      "1932\n",
      "1933\n",
      "1934\n",
      "1935\n",
      "1936\n",
      "1937\n",
      "1938\n",
      "1939\n",
      "1940\n",
      "1941\n",
      "1942\n",
      "1943\n",
      "1944\n",
      "1945\n",
      "1946\n",
      "1947\n",
      "1948\n",
      "1949\n",
      "1950\n",
      "1951\n",
      "1952\n",
      "1953\n",
      "1954\n",
      "1955\n",
      "1956\n",
      "1957\n",
      "1958\n",
      "1959\n",
      "1960\n",
      "1961\n",
      "1962\n",
      "1963\n",
      "1964\n",
      "1965\n",
      "1966\n",
      "1967\n",
      "1968\n",
      "1969\n",
      "1970\n",
      "1971\n",
      "1972\n",
      "1973\n",
      "1974\n",
      "1975\n",
      "1976\n",
      "1977\n",
      "1978\n",
      "1979\n",
      "1980\n",
      "1981\n",
      "1982\n",
      "1983\n",
      "1984\n",
      "1985\n",
      "1986\n",
      "1987\n",
      "1988\n",
      "1989\n",
      "1990\n",
      "1991\n",
      "1992\n",
      "1993\n",
      "1994\n",
      "1995\n",
      "1996\n",
      "1997\n",
      "1998\n",
      "1999\n",
      "2000\n",
      "2001\n",
      "2002\n",
      "2003\n",
      "2004\n",
      "2005\n",
      "2006\n",
      "2007\n",
      "2008\n",
      "2009\n",
      "2010\n",
      "2011\n",
      "2012\n",
      "2013\n",
      "2014\n",
      "2015\n",
      "2016\n",
      "2017\n",
      "2018\n",
      "2019\n",
      "2020\n",
      "2021\n",
      "2022\n",
      "2023\n",
      "2024\n",
      "2025\n",
      "2026\n",
      "2027\n",
      "2028\n",
      "2029\n",
      "2030\n",
      "2031\n",
      "2032\n",
      "2033\n",
      "2034\n",
      "2035\n",
      "2036\n",
      "2037\n",
      "2038\n",
      "2039\n",
      "2040\n",
      "2041\n",
      "2042\n",
      "2043\n",
      "2044\n",
      "2045\n",
      "2046\n",
      "2047\n",
      "2048\n",
      "2049\n",
      "2050\n",
      "2051\n",
      "2052\n",
      "2053\n",
      "2054\n",
      "2055\n",
      "2056\n",
      "2057\n",
      "2058\n",
      "2059\n",
      "2060\n",
      "2061\n",
      "2062\n",
      "2063\n",
      "2064\n",
      "2065\n",
      "2066\n",
      "2067\n",
      "2068\n",
      "2069\n",
      "2070\n",
      "2071\n",
      "2072\n",
      "2073\n",
      "2074\n",
      "2075\n",
      "2076\n",
      "2077\n",
      "2078\n",
      "2079\n",
      "2080\n",
      "2081\n",
      "2082\n",
      "2083\n",
      "2084\n",
      "2085\n",
      "2086\n",
      "2087\n",
      "2088\n",
      "2089\n",
      "2090\n",
      "2091\n",
      "2092\n",
      "2093\n",
      "2094\n",
      "2095\n",
      "2096\n",
      "2097\n",
      "2098\n",
      "2099\n",
      "2100\n",
      "2101\n",
      "2102\n",
      "2103\n",
      "2104\n",
      "2105\n",
      "2106\n",
      "2107\n",
      "2108\n",
      "2109\n",
      "2110\n",
      "2111\n",
      "2112\n",
      "2113\n",
      "2114\n",
      "2115\n",
      "2116\n",
      "2117\n",
      "2118\n",
      "2119\n",
      "2120\n",
      "2121\n",
      "2122\n",
      "2123\n",
      "2124\n",
      "2125\n",
      "2126\n",
      "2127\n",
      "2128\n",
      "2129\n",
      "2130\n",
      "2131\n",
      "2132\n",
      "2133\n",
      "2134\n",
      "2135\n",
      "2136\n",
      "2137\n",
      "2138\n",
      "2139\n",
      "2140\n",
      "2141\n",
      "2142\n",
      "2143\n",
      "2144\n",
      "2145\n",
      "2146\n",
      "2147\n",
      "2148\n",
      "2149\n",
      "2150\n",
      "2151\n",
      "2152\n",
      "2153\n",
      "2154\n",
      "2155\n",
      "2156\n",
      "2157\n",
      "2158\n",
      "2159\n",
      "2160\n",
      "2161\n",
      "2162\n",
      "2163\n",
      "2164\n",
      "2165\n",
      "2166\n",
      "2167\n",
      "2168\n",
      "2169\n",
      "2170\n",
      "2171\n",
      "2172\n",
      "2173\n",
      "2174\n",
      "2175\n",
      "2176\n",
      "2177\n",
      "2178\n",
      "2179\n",
      "2180\n",
      "2181\n",
      "2182\n",
      "2183\n",
      "2184\n",
      "2185\n",
      "2186\n",
      "2187\n",
      "2188\n",
      "2189\n",
      "2190\n",
      "2191\n",
      "2192\n",
      "2193\n",
      "2194\n",
      "2195\n",
      "2196\n",
      "2197\n",
      "2198\n",
      "2199\n",
      "2200\n",
      "2201\n",
      "2202\n",
      "2203\n",
      "2204\n",
      "2205\n",
      "2206\n",
      "2207\n",
      "2208\n",
      "2209\n",
      "2210\n",
      "2211\n",
      "2212\n",
      "2213\n",
      "2214\n",
      "2215\n",
      "2216\n",
      "2217\n",
      "2218\n",
      "2219\n",
      "2220\n",
      "2221\n",
      "2222\n",
      "2223\n",
      "2224\n",
      "2225\n",
      "2226\n",
      "2227\n",
      "2228\n",
      "2229\n",
      "2230\n",
      "2231\n",
      "2232\n",
      "2233\n",
      "2234\n",
      "2235\n",
      "2236\n",
      "2237\n",
      "2238\n",
      "2239\n",
      "2240\n",
      "2241\n",
      "2242\n",
      "2243\n",
      "2244\n",
      "2245\n",
      "2246\n",
      "2247\n",
      "2248\n",
      "2249\n",
      "2250\n",
      "2251\n",
      "2252\n",
      "2253\n",
      "2254\n",
      "2255\n",
      "2256\n",
      "2257\n",
      "2258\n",
      "2259\n",
      "2260\n",
      "2261\n",
      "2262\n",
      "2263\n",
      "2264\n",
      "2265\n",
      "2266\n",
      "2267\n",
      "2268\n",
      "2269\n",
      "2270\n",
      "2271\n",
      "2272\n",
      "2273\n",
      "2274\n",
      "2275\n",
      "2276\n",
      "2277\n",
      "2278\n",
      "2279\n",
      "2280\n",
      "2281\n",
      "2282\n",
      "2283\n",
      "2284\n",
      "2285\n",
      "2286\n",
      "2287\n",
      "2288\n",
      "2289\n",
      "2290\n",
      "2291\n",
      "2292\n",
      "2293\n",
      "2294\n",
      "2295\n",
      "2296\n",
      "2297\n",
      "2298\n",
      "2299\n",
      "2300\n",
      "2301\n",
      "2302\n",
      "2303\n",
      "2304\n",
      "2305\n",
      "2306\n",
      "2307\n",
      "2308\n",
      "2309\n",
      "2310\n",
      "2311\n",
      "2312\n",
      "2313\n",
      "2314\n",
      "2315\n",
      "2316\n",
      "2317\n",
      "2318\n",
      "2319\n",
      "2320\n",
      "2321\n",
      "2322\n",
      "2323\n",
      "2324\n",
      "2325\n",
      "2326\n",
      "2327\n",
      "2328\n",
      "2329\n",
      "2330\n",
      "2331\n",
      "2332\n",
      "2333\n",
      "2334\n",
      "2335\n",
      "2336\n",
      "2337\n",
      "2338\n",
      "2339\n",
      "2340\n",
      "2341\n",
      "2342\n",
      "2343\n",
      "2344\n",
      "2345\n",
      "2346\n",
      "2347\n",
      "2348\n",
      "2349\n",
      "2350\n",
      "2351\n",
      "2352\n",
      "2353\n",
      "2354\n",
      "2355\n",
      "2356\n",
      "2357\n",
      "2358\n",
      "2359\n",
      "2360\n",
      "2361\n",
      "2362\n",
      "2363\n",
      "2364\n",
      "2365\n",
      "2366\n",
      "2367\n",
      "2368\n",
      "2369\n",
      "2370\n",
      "2371\n",
      "2372\n",
      "2373\n",
      "2374\n",
      "2375\n",
      "2376\n",
      "2377\n",
      "2378\n",
      "2379\n",
      "2380\n",
      "2381\n",
      "2382\n",
      "2383\n",
      "2384\n",
      "2385\n",
      "2386\n",
      "2387\n",
      "2388\n",
      "2389\n",
      "2390\n",
      "2391\n",
      "2392\n",
      "2393\n",
      "2394\n",
      "2395\n",
      "2396\n",
      "2397\n",
      "2398\n",
      "2399\n",
      "2400\n",
      "2401\n",
      "2402\n",
      "2403\n",
      "2404\n",
      "2405\n",
      "2406\n",
      "2407\n",
      "2408\n",
      "2409\n",
      "2410\n",
      "2411\n",
      "2412\n",
      "2413\n",
      "2414\n",
      "2415\n",
      "2416\n",
      "2417\n",
      "2418\n",
      "2419\n",
      "2420\n",
      "2421\n",
      "2422\n",
      "2423\n",
      "2424\n",
      "2425\n",
      "2426\n",
      "2427\n",
      "2428\n",
      "2429\n",
      "2430\n",
      "2431\n",
      "2432\n",
      "2433\n",
      "2434\n",
      "2435\n",
      "2436\n",
      "2437\n",
      "2438\n",
      "2439\n",
      "2440\n",
      "2441\n",
      "2442\n",
      "2443\n",
      "2444\n",
      "2445\n",
      "2446\n",
      "2447\n",
      "2448\n",
      "2449\n",
      "2450\n",
      "2451\n",
      "2452\n",
      "2453\n",
      "2454\n",
      "2455\n",
      "2456\n",
      "2457\n",
      "2458\n",
      "2459\n",
      "2460\n",
      "2461\n",
      "2462\n",
      "2463\n",
      "2464\n",
      "2465\n",
      "2466\n",
      "2467\n",
      "2468\n",
      "2469\n",
      "2470\n",
      "2471\n",
      "2472\n",
      "2473\n",
      "2474\n",
      "2475\n",
      "2476\n",
      "2477\n",
      "2478\n",
      "2479\n",
      "2480\n",
      "2481\n",
      "2482\n",
      "2483\n",
      "2484\n",
      "2485\n",
      "2486\n",
      "2487\n",
      "2488\n",
      "2489\n",
      "2490\n",
      "2491\n",
      "2492\n",
      "2493\n",
      "2494\n",
      "2495\n",
      "2496\n",
      "2497\n",
      "2498\n",
      "2499\n",
      "2500\n",
      "2501\n",
      "2502\n",
      "2503\n",
      "2504\n",
      "2505\n",
      "2506\n",
      "2507\n",
      "2508\n",
      "2509\n",
      "2510\n",
      "2511\n",
      "2512\n",
      "2513\n",
      "2514\n",
      "2515\n",
      "2516\n",
      "2517\n",
      "2518\n",
      "2519\n",
      "2520\n",
      "2521\n",
      "2522\n",
      "2523\n",
      "2524\n",
      "2525\n",
      "2526\n",
      "2527\n",
      "2528\n",
      "2529\n",
      "2530\n",
      "2531\n",
      "2532\n",
      "2533\n",
      "2534\n",
      "2535\n",
      "2536\n",
      "2537\n",
      "2538\n",
      "2539\n",
      "2540\n",
      "2541\n",
      "2542\n",
      "2543\n",
      "2544\n",
      "2545\n",
      "2546\n",
      "2547\n",
      "2548\n",
      "2549\n",
      "2550\n",
      "2551\n",
      "2552\n",
      "2553\n",
      "2554\n",
      "2555\n",
      "2556\n",
      "2557\n",
      "2558\n",
      "2559\n",
      "2560\n",
      "2561\n",
      "2562\n",
      "2563\n",
      "2564\n",
      "2565\n",
      "2566\n",
      "2567\n",
      "2568\n",
      "2569\n",
      "2570\n",
      "2571\n",
      "2572\n",
      "2573\n",
      "2574\n",
      "2575\n",
      "2576\n",
      "2577\n",
      "2578\n",
      "2579\n",
      "2580\n",
      "2581\n",
      "2582\n",
      "2583\n",
      "2584\n",
      "2585\n",
      "2586\n",
      "2587\n",
      "2588\n",
      "2589\n",
      "2590\n",
      "2591\n",
      "2592\n",
      "2593\n",
      "2594\n",
      "2595\n",
      "2596\n",
      "2597\n",
      "2598\n",
      "2599\n",
      "2600\n",
      "2601\n",
      "2602\n",
      "2603\n",
      "2604\n",
      "2605\n",
      "2606\n",
      "2607\n",
      "2608\n",
      "2609\n",
      "2610\n",
      "2611\n",
      "2612\n",
      "2613\n",
      "2614\n",
      "2615\n",
      "2616\n",
      "2617\n",
      "2618\n",
      "2619\n",
      "2620\n",
      "2621\n",
      "2622\n",
      "2623\n",
      "2624\n",
      "2625\n",
      "2626\n",
      "2627\n",
      "2628\n",
      "2629\n",
      "2630\n",
      "2631\n",
      "2632\n",
      "2633\n",
      "2634\n",
      "2635\n",
      "2636\n",
      "2637\n",
      "2638\n",
      "2639\n",
      "2640\n",
      "2641\n",
      "2642\n",
      "2643\n",
      "2644\n",
      "2645\n",
      "2646\n",
      "2647\n",
      "2648\n",
      "2649\n",
      "2650\n",
      "2651\n",
      "2652\n",
      "2653\n",
      "2654\n",
      "2655\n",
      "2656\n",
      "2657\n",
      "2658\n",
      "2659\n",
      "2660\n",
      "2661\n",
      "2662\n",
      "2663\n",
      "2664\n",
      "2665\n",
      "2666\n",
      "2667\n",
      "2668\n",
      "2669\n",
      "2670\n",
      "2671\n",
      "2672\n",
      "2673\n",
      "2674\n",
      "2675\n",
      "2676\n",
      "2677\n",
      "2678\n",
      "2679\n",
      "2680\n",
      "2681\n",
      "2682\n",
      "2683\n",
      "2684\n",
      "2685\n",
      "2686\n",
      "2687\n",
      "2688\n",
      "2689\n",
      "2690\n",
      "2691\n",
      "2692\n",
      "2693\n",
      "2694\n",
      "2695\n",
      "2696\n",
      "2697\n",
      "2698\n",
      "2699\n",
      "2700\n",
      "2701\n",
      "2702\n",
      "2703\n",
      "2704\n",
      "2705\n",
      "2706\n",
      "2707\n",
      "2708\n",
      "2709\n",
      "2710\n",
      "2711\n",
      "2712\n",
      "2713\n",
      "2714\n",
      "2715\n",
      "2716\n",
      "2717\n",
      "2718\n",
      "2719\n",
      "2720\n",
      "2721\n",
      "2722\n",
      "2723\n",
      "2724\n",
      "2725\n",
      "2726\n",
      "2727\n",
      "2728\n",
      "2729\n",
      "2730\n",
      "2731\n",
      "2732\n",
      "2733\n",
      "2734\n",
      "2735\n",
      "2736\n",
      "2737\n",
      "2738\n",
      "2739\n",
      "2740\n",
      "2741\n",
      "2742\n",
      "2743\n",
      "2744\n",
      "2745\n",
      "2746\n",
      "2747\n",
      "2748\n",
      "2749\n",
      "2750\n",
      "2751\n",
      "2752\n",
      "2753\n",
      "2754\n",
      "2755\n",
      "2756\n",
      "2757\n",
      "2758\n",
      "2759\n",
      "2760\n",
      "2761\n",
      "2762\n",
      "2763\n",
      "2764\n",
      "2765\n",
      "2766\n",
      "2767\n",
      "2768\n",
      "2769\n",
      "2770\n",
      "2771\n",
      "2772\n",
      "2773\n",
      "2774\n",
      "2775\n",
      "2776\n",
      "2777\n",
      "2778\n",
      "2779\n",
      "2780\n",
      "2781\n",
      "2782\n",
      "2783\n",
      "2784\n",
      "2785\n",
      "2786\n",
      "2787\n",
      "2788\n",
      "2789\n",
      "2790\n",
      "2791\n",
      "2792\n",
      "2793\n",
      "2794\n",
      "2795\n",
      "2796\n",
      "2797\n",
      "2798\n",
      "2799\n",
      "2800\n",
      "2801\n",
      "2802\n",
      "2803\n",
      "2804\n",
      "2805\n",
      "2806\n",
      "2807\n",
      "2808\n",
      "2809\n",
      "2810\n",
      "2811\n",
      "2812\n",
      "2813\n",
      "2814\n",
      "2815\n",
      "2816\n",
      "2817\n",
      "2818\n",
      "2819\n",
      "2820\n",
      "2821\n",
      "2822\n",
      "2823\n",
      "2824\n",
      "2825\n",
      "2826\n",
      "2827\n",
      "2828\n",
      "2829\n",
      "2830\n",
      "2831\n",
      "2832\n",
      "2833\n",
      "2834\n",
      "2835\n",
      "2836\n",
      "2837\n",
      "2838\n",
      "2839\n",
      "2840\n",
      "2841\n",
      "2842\n",
      "2843\n",
      "2844\n",
      "2845\n",
      "2846\n",
      "2847\n",
      "2848\n",
      "2849\n",
      "2850\n",
      "2851\n",
      "2852\n",
      "2853\n",
      "2854\n",
      "2855\n",
      "2856\n",
      "2857\n",
      "2858\n",
      "2859\n",
      "2860\n",
      "2861\n",
      "2862\n",
      "2863\n",
      "2864\n",
      "2865\n",
      "2866\n",
      "2867\n",
      "2868\n",
      "2869\n",
      "2870\n",
      "2871\n",
      "2872\n",
      "2873\n",
      "2874\n",
      "2875\n",
      "2876\n",
      "2877\n",
      "2878\n",
      "2879\n",
      "2880\n",
      "2881\n",
      "2882\n",
      "2883\n",
      "2884\n",
      "2885\n",
      "2886\n",
      "2887\n",
      "2888\n",
      "2889\n",
      "2890\n",
      "2891\n",
      "2892\n",
      "2893\n",
      "2894\n",
      "2895\n",
      "2896\n",
      "2897\n",
      "2898\n",
      "2899\n",
      "2900\n",
      "2901\n",
      "2902\n",
      "2903\n",
      "2904\n",
      "2905\n",
      "2906\n",
      "2907\n",
      "2908\n",
      "2909\n",
      "2910\n",
      "2911\n",
      "2912\n",
      "2913\n",
      "2914\n",
      "2915\n",
      "2916\n",
      "2917\n",
      "2918\n",
      "2919\n",
      "2920\n",
      "2921\n",
      "2922\n",
      "2923\n",
      "2924\n",
      "2925\n",
      "2926\n",
      "2927\n",
      "2928\n",
      "2929\n",
      "2930\n",
      "2931\n",
      "2932\n",
      "2933\n",
      "2934\n",
      "2935\n",
      "2936\n",
      "2937\n",
      "2938\n",
      "2939\n",
      "2940\n",
      "2941\n",
      "2942\n",
      "2943\n",
      "2944\n",
      "2945\n",
      "2946\n",
      "2947\n",
      "2948\n",
      "2949\n",
      "2950\n",
      "2951\n",
      "2952\n",
      "2953\n",
      "2954\n",
      "2955\n",
      "2956\n",
      "2957\n",
      "2958\n",
      "2959\n",
      "2960\n",
      "2961\n",
      "2962\n",
      "2963\n",
      "2964\n",
      "2965\n",
      "2966\n",
      "2967\n",
      "2968\n",
      "2969\n",
      "2970\n",
      "2971\n",
      "2972\n",
      "2973\n",
      "2974\n",
      "2975\n",
      "2976\n",
      "2977\n",
      "2978\n",
      "2979\n",
      "2980\n",
      "2981\n",
      "2982\n",
      "2983\n",
      "2984\n",
      "2985\n",
      "2986\n",
      "2987\n",
      "2988\n",
      "2989\n",
      "2990\n",
      "2991\n",
      "2992\n",
      "2993\n",
      "2994\n",
      "2995\n",
      "2996\n",
      "2997\n",
      "2998\n",
      "2999\n",
      "3000\n",
      "3001\n",
      "3002\n",
      "3003\n",
      "3004\n",
      "3005\n",
      "3006\n",
      "3007\n",
      "3008\n",
      "3009\n",
      "3010\n",
      "3011\n",
      "3012\n",
      "3013\n",
      "3014\n",
      "3015\n",
      "3016\n",
      "3017\n",
      "3018\n",
      "3019\n",
      "3020\n",
      "3021\n",
      "3022\n",
      "3023\n",
      "3024\n",
      "3025\n",
      "3026\n",
      "3027\n",
      "3028\n",
      "3029\n",
      "3030\n",
      "3031\n",
      "3032\n",
      "3033\n",
      "3034\n",
      "3035\n",
      "3036\n",
      "3037\n",
      "3038\n",
      "3039\n",
      "3040\n",
      "3041\n",
      "3042\n",
      "3043\n",
      "3044\n",
      "3045\n",
      "3046\n",
      "3047\n",
      "3048\n",
      "3049\n",
      "3050\n",
      "3051\n",
      "3052\n",
      "3053\n",
      "3054\n",
      "3055\n",
      "3056\n",
      "3057\n",
      "3058\n",
      "3059\n",
      "3060\n",
      "3061\n",
      "3062\n",
      "3063\n",
      "3064\n",
      "3065\n",
      "3066\n",
      "3067\n",
      "3068\n",
      "3069\n",
      "3070\n",
      "3071\n",
      "3072\n",
      "3073\n",
      "3074\n",
      "3075\n",
      "3076\n",
      "3077\n",
      "3078\n",
      "3079\n",
      "3080\n",
      "3081\n",
      "3082\n",
      "3083\n",
      "3084\n",
      "3085\n",
      "3086\n",
      "3087\n",
      "3088\n",
      "3089\n",
      "3090\n",
      "3091\n",
      "3092\n",
      "3093\n",
      "3094\n",
      "3095\n",
      "3096\n",
      "3097\n",
      "3098\n",
      "3099\n",
      "3100\n",
      "3101\n",
      "3102\n",
      "3103\n",
      "3104\n",
      "3105\n",
      "3106\n",
      "3107\n",
      "3108\n",
      "3109\n",
      "3110\n",
      "3111\n",
      "3112\n",
      "3113\n",
      "3114\n",
      "3115\n",
      "3116\n",
      "3117\n",
      "3118\n",
      "3119\n",
      "3120\n",
      "3121\n",
      "3122\n",
      "3123\n",
      "3124\n",
      "3125\n",
      "3126\n"
     ]
    }
   ],
   "source": [
    "i=0\n",
    "for f_ind in feature_importance[index_fs]:\n",
    "    i+=1\n",
    "    if(f_ind!=0):\n",
    "        print(i)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#基模型的特征选择\n",
    "clf_fs = ensemble.GradientBoostingRegressor(random_state=20)\n",
    "clf_fs =clf_fs.fit(dg_scaled_train, score_train)\n",
    "clf_fs.feature_importances_  \n",
    "\n",
    "model_fs = SelectFromModel(clf_fs, prefit=True)\n",
    "X = model_fs.transform(dg_scaled_train)\n",
    "\n",
    "\n",
    "#SelectFromModel(ensemble.GradientBoostingRegressor()).fit_transform(dg_scaled_train, score_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_predict=model_fs.transform(dg_scaled_test)\n",
    "print(dg_scaled_test)\n",
    "np.shape(X_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.73126432, -0.34315163,  2.96723984, ..., -0.91859817,\n",
       "         0.34010331, -0.33148782],\n",
       "       [ 1.78436303, -0.34315163,  0.2107044 , ..., -0.26914594,\n",
       "         0.64388838, -0.33148782],\n",
       "       [ 0.76068048, -0.34315163, -1.46411367, ...,  2.56679545,\n",
       "        -1.03016126, -0.33148782],\n",
       "       ...,\n",
       "       [ 0.49593499, -0.34315163, -0.63438729, ...,  0.27206425,\n",
       "         0.5469357 ,  2.78608699],\n",
       "       [-0.62776252,  2.18560956, -1.08305415, ..., -0.55057524,\n",
       "         0.19790605, -0.33148782],\n",
       "       [-0.23946914, -0.34315163,  0.33669989, ..., -1.04848862,\n",
       "        -0.24161278, -0.33148782]])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.51124829,  0.87139512, -1.16342225, ...,  0.39072476,\n",
       "        -0.15234158, -0.71373671],\n",
       "       [-0.44935252,  1.80973467,  1.73707351, ..., -0.86456136,\n",
       "        -1.00349245,  0.22295745],\n",
       "       [-0.78634061,  0.54021645,  0.68127279, ..., -0.14725501,\n",
       "        -1.62339108,  1.47188301],\n",
       "       ...,\n",
       "       [-0.53875752, -0.01174799,  0.3028738 , ...,  0.23701625,\n",
       "         1.90329051,  0.22295745],\n",
       "       [ 0.8504587 , -1.36406087,  2.43136807, ...,  0.85185027,\n",
       "         3.66552767,  0.22295745],\n",
       "       [-0.24303328, -0.17733732, -1.20903284, ...,  1.28735769,\n",
       "        -1.23881962, -0.71373671]])"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13731, 380)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10984, 380), (10984,), (2747, 380), (2747,))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#划分数据集\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, score_train, test_size=0.2, random_state=21)\n",
    "X_train.shape,y_train.shape,X_test.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2、学習（train）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5897356830942392\n",
      "0.5897356830942392\n"
     ]
    }
   ],
   "source": [
    "#SVR模型实现\n",
    "model_svc=svm.SVR()\n",
    "model_svc.fit(X_train,y_train)\n",
    "\n",
    "print(model_svc.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_svc.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GBDT 0.5903171534507402<br>\n",
    "RF 0.5714306438983159<br >\n",
    "XBG 0.5864080375808052<br>\n",
    "平均500 0.5877608712908593<br>\n",
    "平均400 0.5889615497881555<br>\n",
    "358 0.5888839063663873<br>\n",
    "370 0.5889239406579279<br>\n",
    "375 0.589024306528543<br>\n",
    "380 0.5897356830942392<br>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "SVR(C=10.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma=0.001,\n",
      "    kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
      "0.379(+/-0.007) for {'C': 1.0, 'gamma': 0.0001}\n",
      "0.502(+/-0.009) for {'C': 1.0, 'gamma': 0.001}\n",
      "0.493(+/-0.006) for {'C': 1.0, 'gamma': 0.01}\n",
      "0.035(+/-0.001) for {'C': 1.0, 'gamma': 0.1}\n",
      "0.010(+/-0.001) for {'C': 1.0, 'gamma': 1.0}\n",
      "0.441(+/-0.010) for {'C': 10.0, 'gamma': 0.0001}\n",
      "0.537(+/-0.010) for {'C': 10.0, 'gamma': 0.001}\n",
      "0.502(+/-0.008) for {'C': 10.0, 'gamma': 0.01}\n",
      "0.040(+/-0.002) for {'C': 10.0, 'gamma': 0.1}\n",
      "0.011(+/-0.001) for {'C': 10.0, 'gamma': 1.0}\n",
      "0.492(+/-0.010) for {'C': 100.0, 'gamma': 0.0001}\n",
      "0.479(+/-0.011) for {'C': 100.0, 'gamma': 0.001}\n",
      "0.503(+/-0.008) for {'C': 100.0, 'gamma': 0.01}\n",
      "0.040(+/-0.002) for {'C': 100.0, 'gamma': 0.1}\n",
      "0.010(+/-0.002) for {'C': 100.0, 'gamma': 1.0}\n",
      "0.496(+/-0.005) for {'C': 1000.0, 'gamma': 0.0001}\n",
      "0.456(+/-0.012) for {'C': 1000.0, 'gamma': 0.001}\n",
      "0.499(+/-0.016) for {'C': 1000.0, 'gamma': 0.01}\n",
      "0.039(+/-0.003) for {'C': 1000.0, 'gamma': 0.1}\n",
      "0.007(+/-0.007) for {'C': 1000.0, 'gamma': 1.0}\n"
     ]
    }
   ],
   "source": [
    "#SVR模型调参数：C gamma\n",
    "param_g=[{\"C\": [1e0, 1e1, 1e2, 1e3],\"gamma\": np.logspace(-4, 0, 5)}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf=GridSearchCV(svm.SVR(),param_grid=param_g,cv=3,scoring=score)\n",
    "    clf.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf.best_estimator_)\n",
    "    \n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds=clf.cv_results_['std_test_score']\n",
    "    params = clf.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5940389333631619\n",
      "0.5940389333631619\n"
     ]
    }
   ],
   "source": [
    "#SVR模型实现\n",
    "model_svc_t=clf.best_estimator_\n",
    "model_svc_t.fit(X_train,y_train)\n",
    "\n",
    "print(model_svc_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_svc_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_svc=clf.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4808578016857748\n",
      "0.4808578016857748\n"
     ]
    }
   ],
   "source": [
    "model_svcRF=RandomForestRegressor(random_state=160)\n",
    "model_svcRF.fit(X_train,y_train)\n",
    "\n",
    "print(model_svcRF.score(X_test,y_test))#0.4697676007383277\n",
    "\n",
    "y_pred = model_svcRF.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "380 0.4808578016857748"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "                      max_features='auto', max_leaf_nodes=None,\n",
      "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                      min_samples_leaf=1, min_samples_split=2,\n",
      "                      min_weight_fraction_leaf=0.0, n_estimators=500,\n",
      "                      n_jobs=None, oob_score=False, random_state=160, verbose=0,\n",
      "                      warm_start=False)\n",
      "0.475(+/-0.014) for {'n_estimators': 20}\n",
      "0.498(+/-0.012) for {'n_estimators': 50}\n",
      "0.505(+/-0.012) for {'n_estimators': 100}\n",
      "0.507(+/-0.013) for {'n_estimators': 150}\n",
      "0.508(+/-0.013) for {'n_estimators': 200}\n",
      "0.509(+/-0.013) for {'n_estimators': 300}\n",
      "0.510(+/-0.012) for {'n_estimators': 400}\n",
      "0.510(+/-0.012) for {'n_estimators': 500}\n"
     ]
    }
   ],
   "source": [
    "N_estimators=[{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf=GridSearchCV(RandomForestRegressor(random_state=160),param_grid=N_estimators,cv=3,scoring=score)\n",
    "    clf.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf.best_estimator_)\n",
    "    \n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds=clf.cv_results_['std_test_score']\n",
    "    params = clf.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5451068944441522\n",
      "0.5451068944441522\n"
     ]
    }
   ],
   "source": [
    "model_svcRF_t=clf.best_estimator_\n",
    "model_svcRF_t.fit(X_train,y_train)\n",
    "\n",
    "print(model_svcRF_t.score(X_test,y_test))#默认0.4697676007383277\n",
    "\n",
    "y_pred = model_svcRF_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "model_svcRF=clf.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[02:15:34] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "0.4852452621853672\n",
      "0.4852452621853672\n"
     ]
    }
   ],
   "source": [
    "model_xgbr = xgb.XGBRegressor(random_state=161)\n",
    "model_xgbr.fit(X_train, y_train)\n",
    "\n",
    "print(model_xgbr.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_xgbr.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GBDT 0.48840397626857135<br>\n",
    "RF 0.47431755406122006<br />\n",
    "XGB 0.4886366445001801<br>\n",
    "平均500 0.4851985990179926<br>\n",
    "平均400 0.4858654940215366<br>\n",
    "358 0.4859184552814983<br>\n",
    "370 0.48736871205486315<br>\n",
    "375 0.4852452621853672<br>\n",
    "380 0.4852452621853672<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[02:19:48] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
      "             max_depth=3, min_child_weight=1, missing=None, n_estimators=500,\n",
      "             n_jobs=1, nthread=None, objective='reg:linear', random_state=161,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "             silent=None, subsample=1, verbosity=1)\n",
      "0.297(+/-0.011) for {'n_estimators': 20}\n",
      "0.423(+/-0.011) for {'n_estimators': 50}\n",
      "0.471(+/-0.010) for {'n_estimators': 100}\n",
      "0.496(+/-0.010) for {'n_estimators': 150}\n",
      "0.508(+/-0.011) for {'n_estimators': 200}\n",
      "0.523(+/-0.009) for {'n_estimators': 300}\n",
      "0.533(+/-0.009) for {'n_estimators': 400}\n",
      "0.538(+/-0.008) for {'n_estimators': 500}\n"
     ]
    }
   ],
   "source": [
    "#调参 XGboost——n_estimators 其他默认\n",
    "cv_params = [{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "\n",
    "scores_GBM=['r2']\n",
    "\n",
    "for score in scores_GBM:\n",
    "    print(score)\n",
    "    model = XGBRegressor(random_state=161)\n",
    "    optimized_GBM=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM.best_estimator_)\n",
    "    \n",
    "    means_GBM = optimized_GBM.cv_results_['mean_test_score']\n",
    "    stds_GBM=optimized_GBM.cv_results_['std_test_score']\n",
    "    params_GBM = optimized_GBM.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM,mean_score_GBM,std_score_GBM) in zip(params_GBM,means_GBM,stds_GBM):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM,std_score_GBM,param_GBM))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "Fitting 3 folds for each of 12 candidates, totalling 36 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  36 out of  36 | elapsed: 24.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[02:45:24] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
      "             max_depth=7, min_child_weight=1, missing=None, n_estimators=500,\n",
      "             n_jobs=1, nthread=None, objective='reg:linear', random_state=161,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "             silent=None, subsample=1, verbosity=1)\n",
      "0.538(+/-0.008) for {'max_depth': 3, 'min_child_weight': 1}\n",
      "0.537(+/-0.009) for {'max_depth': 3, 'min_child_weight': 3}\n",
      "0.537(+/-0.008) for {'max_depth': 3, 'min_child_weight': 5}\n",
      "0.557(+/-0.009) for {'max_depth': 5, 'min_child_weight': 1}\n",
      "0.555(+/-0.010) for {'max_depth': 5, 'min_child_weight': 3}\n",
      "0.556(+/-0.011) for {'max_depth': 5, 'min_child_weight': 5}\n",
      "0.561(+/-0.009) for {'max_depth': 7, 'min_child_weight': 1}\n",
      "0.558(+/-0.011) for {'max_depth': 7, 'min_child_weight': 3}\n",
      "0.555(+/-0.009) for {'max_depth': 7, 'min_child_weight': 5}\n",
      "0.544(+/-0.013) for {'max_depth': 9, 'min_child_weight': 1}\n",
      "0.550(+/-0.012) for {'max_depth': 9, 'min_child_weight': 3}\n",
      "0.549(+/-0.009) for {'max_depth': 9, 'min_child_weight': 5}\n"
     ]
    }
   ],
   "source": [
    "#调参 XGboost——maxdepth minchildweight 其他默认\n",
    "cv_params = [{'max_depth': [3, 5,7,9], 'min_child_weight': [1, 3, 5]}]\n",
    "\n",
    "scores_GBM_mami=['r2']\n",
    "\n",
    "for score in scores_GBM_mami:\n",
    "    print(score)\n",
    "    model = optimized_GBM.best_estimator_\n",
    "    optimized_GBM_mami=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM_mami.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM_mami.best_estimator_)\n",
    "    \n",
    "    means_GBM_mami = optimized_GBM_mami.cv_results_['mean_test_score']\n",
    "    stds_GBM_mami=optimized_GBM_mami.cv_results_['std_test_score']\n",
    "    params_GBM_mami = optimized_GBM_mami.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM_mami,mean_score_GBM_mami,std_score_GBM_mami) in zip(params_GBM_mami,means_GBM_mami,stds_GBM_mami):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM_mami,std_score_GBM_mami,param_GBM_mami))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "Fitting 3 folds for each of 6 candidates, totalling 18 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  18 out of  18 | elapsed: 15.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[03:03:25] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
      "             max_depth=7, min_child_weight=1, missing=None, n_estimators=500,\n",
      "             n_jobs=1, nthread=None, objective='reg:linear', random_state=161,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "             silent=None, subsample=1, verbosity=1)\n",
      "-2.447(+/-0.012) for {'learning_rate': 0.0001}\n",
      "-0.765(+/-0.010) for {'learning_rate': 0.001}\n",
      "0.530(+/-0.012) for {'learning_rate': 0.01}\n",
      "0.561(+/-0.009) for {'learning_rate': 0.1}\n",
      "0.538(+/-0.010) for {'learning_rate': 0.2}\n",
      "0.504(+/-0.009) for {'learning_rate': 0.3}\n"
     ]
    }
   ],
   "source": [
    "#调参 XGboost——学习率\n",
    "cv_params = [{'learning_rate':  [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]}]\n",
    "\n",
    "scores_GBM_lr=['r2']\n",
    "\n",
    "for score in scores_GBM_lr:\n",
    "    print(score)\n",
    "    model = optimized_GBM_mami.best_estimator_\n",
    "    optimized_GBM_lr=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM_lr.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM_lr.best_estimator_)\n",
    "    \n",
    "    means_GBM_lr = optimized_GBM_lr.cv_results_['mean_test_score']\n",
    "    stds_GBM_lr=optimized_GBM_lr.cv_results_['std_test_score']\n",
    "    params_GBM_lr = optimized_GBM_lr.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM_lr,mean_score_GBM_lr,std_score_GBM_lr) in zip(params_GBM_lr,means_GBM_lr,stds_GBM_lr):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM_lr,std_score_GBM_lr,param_GBM_lr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[03:06:11] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "0.5650615677014149\n",
      "0.5650615677014149\n"
     ]
    }
   ],
   "source": [
    "model_xgbr_t = optimized_GBM.best_estimator_\n",
    "model_xgbr_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_xgbr_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_xgbr_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "model_xgbr = optimized_GBM.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.46687902869520853\n",
      "0.46687902869520853\n"
     ]
    }
   ],
   "source": [
    "# 4.kNN回归\n",
    "model_k_neighbor = neighbors.KNeighborsRegressor()\n",
    "model_k_neighbor.fit(X_train, y_train)\n",
    "\n",
    "print(model_k_neighbor.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_k_neighbor.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "Fitting 3 folds for each of 60 candidates, totalling 180 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-f619a960d059>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     15\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mneighbors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mKNeighborsRegressor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m     \u001b[0mclf_k\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mGridSearchCV\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparam_grid\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparam_grids\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m     \u001b[0mclf_k\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclf_k\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_estimator_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[0;32m    685\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    686\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 687\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    688\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    689\u001b[0m         \u001b[1;31m# For multi-metric evaluation, store the best_index_, best_params_ and\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[1;34m(self, evaluate_candidates)\u001b[0m\n\u001b[0;32m   1146\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1147\u001b[0m         \u001b[1;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1148\u001b[1;33m         \u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1149\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1150\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[1;34m(candidate_params)\u001b[0m\n\u001b[0;32m    664\u001b[0m                                \u001b[1;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    665\u001b[0m                                in product(candidate_params,\n\u001b[1;32m--> 666\u001b[1;33m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[0;32m    667\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    668\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m<\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    932\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    933\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 934\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    935\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    936\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    831\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    832\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'supports_timeout'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 833\u001b[1;33m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    834\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    835\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\lib\\site-packages\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[1;34m(future, timeout)\u001b[0m\n\u001b[0;32m    519\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[0;32m    520\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 521\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    522\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mLokyTimeoutError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    523\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\lib\\concurrent\\futures\\_base.py\u001b[0m in \u001b[0;36mresult\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    425\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    426\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 427\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    428\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    429\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mCANCELLED\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCANCELLED_AND_NOTIFIED\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\lib\\threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    293\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 295\u001b[1;33m                 \u001b[0mwaiter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    296\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    297\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "param_grids=[\n",
    "    {\n",
    "        'weights':['uniform'],\n",
    "        'n_neighbors':[i for i in range(1,11)]\n",
    "    },\n",
    "    {\n",
    "        'weights':['distance'],\n",
    "        'n_neighbors':[i for i in range(1,11)],\n",
    "        'p':[i for i in range(1,6)]\n",
    "    }]\n",
    "scores_KNN=['r2']\n",
    "\n",
    "for score in scores_KNN:\n",
    "    print(score)\n",
    "    model = neighbors.KNeighborsRegressor()\n",
    "    clf_k=GridSearchCV(estimator=model, param_grid=param_grids, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    clf_k.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_k.best_estimator_)\n",
    "    \n",
    "    means_KNN = clf_k.cv_results_['mean_test_score']\n",
    "    stds_KNN=clf_k.cv_results_['std_test_score']\n",
    "    params_KNN = clf_k.cv_results_['params']\n",
    "    \n",
    "    for (param_KNN,mean_score_KNN,std_score_KNN) in zip(params_KNN,means_KNN,stds_KNN):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_KNN,std_score_KNN,param_KNN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_k_neighbor_t = clf_k.best_estimator_\n",
    "model_k_neighbor_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_k_neighbor_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_k_neighbor_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_k_neighbor = clf_k.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GBRT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gradient_boosting_regressor = ensemble.GradientBoostingRegressor(random_state=162)  \n",
    "\n",
    "model_gradient_boosting_regressor.fit(X_train, y_train)\n",
    "\n",
    "print(model_gradient_boosting_regressor.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_gradient_boosting_regressor.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_estimators=[{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf_gb=GridSearchCV(ensemble.GradientBoostingRegressor(random_state=162),param_grid=N_estimators,cv=3,scoring=score)\n",
    "    clf_gb.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_gb.best_estimator_)\n",
    "    \n",
    "    means = clf_gb.cv_results_['mean_test_score']\n",
    "    stds=clf_gb.cv_results_['std_test_score']\n",
    "    params = clf_gb.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gradient_boosting_regressor_t = clf_gb.best_estimator_\n",
    "\n",
    "model_gradient_boosting_regressor_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_gradient_boosting_regressor_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_gradient_boosting_regressor_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_gradient_boosting_regressor = clf_gb.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二阶段stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 SVR(C=10, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma=0.001,\n",
      "    kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
      "Fold 0\n",
      "Fold 1\n",
      "Fold 2\n",
      "Fold 3\n",
      "Fold 4\n",
      "1 RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "                      max_features='auto', max_leaf_nodes=None,\n",
      "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                      min_samples_leaf=1, min_samples_split=2,\n",
      "                      min_weight_fraction_leaf=0.0, n_estimators=400,\n",
      "                      n_jobs=None, oob_score=False, random_state=None,\n",
      "                      verbose=0, warm_start=False)\n",
      "Fold 0\n",
      "Fold 1\n",
      "Fold 2\n",
      "Fold 3\n",
      "Fold 4\n",
      "2 XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
      "             max_depth=7, min_child_weight=5, missing=None, n_estimators=500,\n",
      "             n_jobs=1, nthread=None, objective='reg:linear', random_state=161,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "             silent=None, subsample=1, verbosity=1)\n",
      "Fold 0\n",
      "[08:41:45] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Fold 1\n",
      "[08:44:28] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Fold 2\n",
      "[08:47:48] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Fold 3\n",
      "[08:50:22] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Fold 4\n",
      "[08:52:56] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "3 KNeighborsRegressor(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "                    metric_params=None, n_jobs=None, n_neighbors=6, p=1,\n",
      "                    weights='distance')\n",
      "Fold 0\n",
      "Fold 1\n",
      "Fold 2\n",
      "Fold 3\n",
      "Fold 4\n",
      "4 GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
      "                          learning_rate=0.1, loss='ls', max_depth=3,\n",
      "                          max_features=None, max_leaf_nodes=None,\n",
      "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                          min_samples_leaf=1, min_samples_split=2,\n",
      "                          min_weight_fraction_leaf=0.0, n_estimators=500,\n",
      "                          n_iter_no_change=None, presort='auto',\n",
      "                          random_state=None, subsample=1.0, tol=0.0001,\n",
      "                          validation_fraction=0.1, verbose=0, warm_start=False)\n",
      "Fold 0\n",
      "Fold 1\n",
      "Fold 2\n",
      "Fold 3\n",
      "Fold 4\n"
     ]
    }
   ],
   "source": [
    "# '''创建训练的数据集'''\n",
    "#data, target = make_blobs(n_samples=50000, centers=2, random_state=0, cluster_std=0.60)\n",
    " \n",
    "# '''模型融合中使用到的各个单模型'''\n",
    "clfs = [svm.SVR(kernel='rbf',C=10,gamma=0.001),\n",
    "        RandomForestRegressor(n_estimators=400),\n",
    "        xgb.XGBRegressor(max_depth=7,min_child_weight=5,n_estimators=500,random_state=161),\n",
    "        neighbors.KNeighborsRegressor(n_neighbors=6, p=1,weights='distance'),\n",
    "        ensemble.GradientBoostingRegressor(n_estimators=500)\n",
    "       ]\n",
    " \n",
    "#'''切分一部分数据作为测试集'''\n",
    "X=X#训练集 数据\n",
    "X_predict=X_predict#测试集 数据\n",
    "y=score_train#训练集 分数\n",
    "#y_predict = \n",
    "\n",
    "\n",
    "dataset_blend_train = np.zeros((X.shape[0], len(clfs)))#第一轮 保存各个模型在训练集上的预测结果 训练集合个数×模型数\n",
    "dataset_blend_test = np.zeros((X_predict.shape[0], len(clfs)))#第一轮 保存各个模型在测试集上的预测结果 训练集合个数×模型数\n",
    "\n",
    "#'''5折stacking'''\n",
    "n_folds = 5\n",
    "kf = KFold(n_folds,True,22)\n",
    "skf=list(kf.split(X))#X或者y\n",
    "\n",
    "for j, clf in enumerate(clfs):\n",
    "    #'''依次训练各个单模型'''\n",
    "    print(j, clf)\n",
    "    dataset_blend_test_j = np.zeros((X_predict.shape[0], len(skf)))#存目前这个模型上的测试集结果(之后求平均)\n",
    "    for i, (train, test) in enumerate(skf):\n",
    "        #'''使用第i个部分作为预测，剩余的部分来训练模型，获得其预测的输出作为第i部分的新特征。'''\n",
    "        print(\"Fold\", i)\n",
    "        X_train, y_train, X_test, y_test = X[train], y[train], X[test], y[test]\n",
    "        clf.fit(X_train, y_train)\n",
    "        y_submission = clf.predict(X_test)#1fold的预测结果\n",
    "        y_submission=y_submission.flatten()#2维数组转1维 KNN需要 \n",
    "        #y_submission.reshape(len(y_submission),1)#一维数组转二维 可以不加\n",
    "        \n",
    "        dataset_blend_train[test, j] = y_submission#在模型顺序对应的j位置 存1fold的预测结果\n",
    "        dataset_blend_test_j[:, i] = clf.predict(X_predict).flatten()#存该模型该折下的测试集预测结果\n",
    "        \n",
    "    #'''对于测试集，直接用这k个模型的预测值均值作为新的特征。'''\n",
    "    dataset_blend_test[:, j] = dataset_blend_test_j.mean(1)#测试集结果按行取平均后储存\n",
    "    \n",
    "    \n",
    "    #print(\"val auc Score: %f\" % r2_score(y_predict, dataset_blend_test[:, j]))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导出数据集到本地\n",
    "submission_train_1=pd.DataFrame(dataset_blend_train)\n",
    "#submission_train_1.head()\n",
    "submission_train_1.to_csv('dataset_blend_train.csv',index=False)#第一轮训练后 train集合预测得到的score集合 训练集样本数x3个模型\n",
    "\n",
    "submission_test_1=pd.DataFrame(dataset_blend_test)\n",
    "#submission_test_1.head()\n",
    "submission_test_1.to_csv('dataset_blend_test.csv',index=False)#第一轮训练后 test集合预测得到的score集合 测试机样本数x3个模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3、予測（predict）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第二轮 \n",
    "#model_stacking_xgb = xgb.XGBRegressor(random_state=200)\n",
    "model_stacking_svm=svm.SVR()\n",
    "model_stacking_svm.fit(dataset_blend_train, y)\n",
    "y_submission = model_stacking_svm.predict(dataset_blend_test)\n",
    "\n",
    "#生成文件\n",
    "submission_df=pd.DataFrame(data={'Id':id_test,'SalePrice':y_submission})\n",
    "submission_df.head()\n",
    "submission_df.to_csv('baseline_st3_svg.csv',header=False,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#调参数\n",
    "X_train, X_test, y_train, y_test = train_test_split(dataset_blend_train, y, test_size=0.2, random_state=123)\n",
    "print(X_train.shape,y_train.shape,X_test.shape,y_test.shape)\n",
    "\n",
    "#SVR模型调参数：C gamma\n",
    "param_g=[{\"C\": [1e0, 1e1, 1e2, 1e3],\"gamma\": np.logspace(-4, 0, 5)}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf_st=GridSearchCV(svm.SVR(),param_grid=param_g,cv=3,scoring=score)\n",
    "    clf_st.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_st.best_estimator_)\n",
    "    \n",
    "    means = clf_st.cv_results_['mean_test_score']\n",
    "    stds=clf_st.cv_results_['std_test_score']\n",
    "    params = clf_st.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10984, 5) (10984,) (2747, 5) (2747,)\n",
      "0.6225271454983285\n"
     ]
    }
   ],
   "source": [
    "#肯定不准确的预测score\n",
    "X_train, X_test, y_train, y_test = train_test_split(dataset_blend_train, y, test_size=0.2, random_state=156)\n",
    "print(X_train.shape,y_train.shape,X_test.shape,y_test.shape)\n",
    "\n",
    "model_stacking_svm.fit(X_train, y_train)\n",
    "\n",
    "print(model_stacking_svm.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_st_best = clf_st.best_estimator_\n",
    "\n",
    "clf_st_best.fit(dataset_blend_train, y)\n",
    "y_submission = clf_st_best.predict(dataset_blend_test)\n",
    "\n",
    "#生成文件\n",
    "submission_df=pd.DataFrame(data={'Id':id_test,'SalePrice':y_submission})\n",
    "submission_df.head()\n",
    "submission_df.to_csv('baseline_st3_2.csv',header=False,index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}