ver1_4_0924.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2019/09/21 ver.1 模板\n",
    "## 一、文件内容\n",
    "特征选择：GBRT <br />\n",
    "单模型构建：<br />\n",
    " 1：SVR<br />\n",
    " 2：RF<br />\n",
    " 3：XGBoost<br />\n",
    " 4：KNN<br />\n",
    " 5：GBRT<br />\n",
    "Stacking：<br />\n",
    " SVR默认\n",
    "## 二、数据集\n",
    "### 导入数据集变量<br />\n",
    "元数据集 训练集全部：dg_train →训练集 特征数据：dg_train<br />\n",
    "元数据集 测试集全部：dg_test  →测试集 特征数据：dg_test<br />\n",
    "<br />\n",
    "训练集 得分数据：score_train<br />\n",
    "测试集 序号：id_test<br />\n",
    "训练集 序号：id_train\n",
    "<br />\n",
    "### 处理数据集变量<br />\n",
    "标准化处理后的原训练集 特征数据：dg_scaled_train<br />\n",
    "标准化处理后的测试集合 特征数据：dg_scaled_test<br />\n",
    "### 特征选择结果\n",
    "特征选择后的训练集 特征数据：X<br />\n",
    "特征选择后的测试集 特征数据：X_predict<br />\n",
    "### 在X中继续划分数据<br />\n",
    "X_train：0.8比例的原训练集特征 用作训练和验证<br />\n",
    "X_test：0.2比例的原训练集特征 用作测试（模型选择）<br />\n",
    "y_train：0.8比例的原训练集分数 对应X_train<br />\n",
    "y_test：0.2比例的原训练集分数 对应X_test<br />\n",
    "\n",
    "## 三、模型信息\n",
    "\n",
    "\n",
    "变量名|模型名|超参数设置|随机数种子|训练集上预测值\n",
    ":---------:|:------------------:|:-----------------:|:-----------------:|:-------------------------:\n",
    "model_svc|支持向量回归||NA|\n",
    "model_svcRF|随机森林回归||160|\n",
    "model_xgbr|Xgboost回归||161|\n",
    "model_k_neighbor|KNN回归||NA|\n",
    "model_gradient_boosting_regressor|GBRT||162|\n",
    "\n",
    "## 四、随机数信息\n",
    "位置|使用函数|随机数种子\n",
    ":------:|:-------:|:--------:|\n",
    "特征处理|GBRT|20\n",
    "特征处理|train_test_split|21\n",
    "stacking|KFold|22\n",
    "预测|train_test_split|23\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1、前処理（preprocess）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#预设导入\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import joblib\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#机器学习导入\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import svm\n",
    "from sklearn import linear_model\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import explained_variance_score \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBRegressor\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn import ensemble\n",
    "from sklearn import neighbors\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dg_train=pd.read_csv('train.csv')\n",
    "dg_test=pd.read_csv(\"test.csv\")\n",
    "\n",
    "#单独提取ID列和score列\n",
    "id_train=dg_train[\"ID\"].values\n",
    "id_test=dg_test[\"ID\"].values\n",
    "score_train=dg_train[\"Score\"].values\n",
    "\n",
    "del dg_train[\"ID\"]\n",
    "del dg_train[\"Score\"]\n",
    "del dg_test[\"ID\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-8abdb1bd7215>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#correlation matrix\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msubplots\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m15\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m12\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0msns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mheatmap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdg_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcorr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mvmax\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.8\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msquare\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda2\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mcorr\u001b[1;34m(self, method, min_periods)\u001b[0m\n\u001b[0;32m   7487\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   7488\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"pearson\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 7489\u001b[1;33m             \u001b[0mcorrel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlibalgos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnancorr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mensure_float64\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mminp\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmin_periods\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   7490\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"spearman\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   7491\u001b[0m             \u001b[0mcorrel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlibalgos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnancorr_spearman\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mensure_float64\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mminp\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmin_periods\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3IAAAKvCAYAAAAvAP2kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAaa0lEQVR4nO3dX6jn913n8de7GaNQawvOLEhmNAGnW8cixD1ku/TCSrvLJBczN0UyULQSOjcbZdciRJQq8cqWpSDEP7NrqQo2xl7oIJFcaMRFTMkp3Q0mJTBEbQ4RMtZsboqN2X3vxe8kPT05M+ebye+czJvzeMDA7/v7fc7vvC8+nJnnfL/n963uDgAAAHO84+0eAAAAgDdHyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDD7hlxVfa6qXqyqv73G61VVv15VV6rqqar60fWPCQAAwGuWnJH7fJKz13n97iSnt/9cTPKbb30sAAAArmXfkOvuv0ryz9dZcj7J7/XKE0neU1Xft64BAQAA+HbH1vAetyV5fsfx1vZz/7h7YVVdzOqsXd75znf+u/e9731r+PYAAADzfPnLX/6n7j5xI1+7jpCrPZ7rvRZ296Ukl5JkY2OjNzc31/DtAQAA5qmqf7jRr13Hp1ZuJTm14/hkkhfW8L4AAADsYR0hdznJT25/euUHkrzc3W+4rBIAAID12PfSyqr6QpIPJTleVVtJfjnJdyRJd/9WkkeT3JPkSpJvJPnpgxoWAACABSHX3Rf2eb2T/Oe1TQQAAMB1rePSSgAAAA6RkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhmUchV1dmqeraqrlTVA3u8/v1V9XhVfaWqnqqqe9Y/KgAAAMmCkKuqW5I8lOTuJGeSXKiqM7uW/VKSR7r7ziT3JvmNdQ8KAADAypIzcncludLdz3X3K0keTnJ+15pO8j3bj9+d5IX1jQgAAMBOS0LutiTP7zje2n5up19J8rGq2kryaJKf2euNqupiVW1W1ebVq1dvYFwAAACWhFzt8VzvOr6Q5PPdfTLJPUl+v6re8N7dfam7N7p748SJE29+WgAAABaF3FaSUzuOT+aNl07el+SRJOnuv0nyXUmOr2NAAAAAvt2SkHsyyemquqOqbs3qw0wu71rztSQfTpKq+qGsQs61kwAAAAdg35Dr7leT3J/ksSRfzerTKZ+uqger6tz2sk8m+URV/e8kX0jy8e7effklAAAAa3BsyaLufjSrDzHZ+dyndjx+JskH1zsaAAAAe1l0Q3AAAABuHkIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMItCrqrOVtWzVXWlqh64xpqfqKpnqurpqvqD9Y4JAADAa47tt6CqbknyUJL/mGQryZNVdbm7n9mx5nSSX0jywe5+qar+zUENDAAAcNQtOSN3V5Ir3f1cd7+S5OEk53et+USSh7r7pSTp7hfXOyYAAACvWRJytyV5fsfx1vZzO703yXur6q+r6omqOrvXG1XVxararKrNq1ev3tjEAAAAR9ySkKs9nutdx8eSnE7yoSQXkvyPqnrPG76o+1J3b3T3xokTJ97srAAAAGRZyG0lObXj+GSSF/ZY8yfd/a/d/XdJns0q7AAAAFizJSH3ZJLTVXVHVd2a5N4kl3et+eMkP54kVXU8q0stn1vnoAAAAKzsG3Ld/WqS+5M8luSrSR7p7qer6sGqOre97LEkX6+qZ5I8nuTnu/vrBzU0AADAUVbdu3/d7XBsbGz05ubm2/K9AQAA3m5V9eXu3riRr110Q3AAAABuHkIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgmEUhV1Vnq+rZqrpSVQ9cZ91Hq6qramN9IwIAALDTviFXVbckeSjJ3UnOJLlQVWf2WPeuJD+b5EvrHhIAAIBvWXJG7q4kV7r7ue5+JcnDSc7vse5Xk3w6yb+scT4AAAB2WRJytyV5fsfx1vZzr6uqO5Oc6u4/vd4bVdXFqtqsqs2rV6++6WEBAABYFnK1x3P9+otV70jy2SSf3O+NuvtSd29098aJEyeWTwkAAMDrloTcVpJTO45PJnlhx/G7krw/yV9W1d8n+UCSyz7wBAAA4GAsCbknk5yuqjuq6tYk9ya5/NqL3f1ydx/v7tu7+/YkTyQ5192bBzIxAADAEbdvyHX3q0nuT/JYkq8meaS7n66qB6vq3EEPCAAAwLc7tmRRdz+a5NFdz33qGms/9NbHAgAA4FoW3RAcAACAm4eQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMMyikKuqs1X1bFVdqaoH9nj956rqmap6qqr+vKp+YP2jAgAAkCwIuaq6JclDSe5OcibJhao6s2vZV5JsdPePJPlikk+ve1AAAABWlpyRuyvJle5+rrtfSfJwkvM7F3T34939je3DJ5KcXO+YAAAAvGZJyN2W5Pkdx1vbz13LfUn+bK8XqupiVW1W1ebVq1eXTwkAAMDrloRc7fFc77mw6mNJNpJ8Zq/Xu/tSd29098aJEyeWTwkAAMDrji1Ys5Xk1I7jk0le2L2oqj6S5BeT/Fh3f3M94wEAALDbkjNyTyY5XVV3VNWtSe5Ncnnngqq6M8lvJznX3S+uf0wAAABes2/IdferSe5P8liSryZ5pLufrqoHq+rc9rLPJPnuJH9UVf+rqi5f4+0AAAB4i5ZcWpnufjTJo7ue+9SOxx9Z81wAAABcw6IbggMAAHDzEHIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABhGyAEAAAwj5AAAAIYRcgAAAMMIOQAAgGGEHAAAwDBCDgAAYBghBwAAMIyQAwAAGEbIAQAADCPkAAAAhhFyAAAAwwg5AACAYYQcAADAMEIOAABgGCEHAAAwjJADAAAYRsgBAAAMI+QAAACGEXIAAADDLAq5qjpbVc9W1ZWqemCP17+zqv5w+/UvVdXt6x4UAACAlX1DrqpuSfJQkruTnElyoarO7Fp2X5KXuvsHk3w2ya+te1AAAABWlpyRuyvJle5+rrtfSfJwkvO71pxP8rvbj7+Y5MNVVesbEwAAgNccW7DmtiTP7zjeSvLvr7Wmu1+tqpeTfG+Sf9q5qKouJrm4ffjNqvrbGxkaDtjx7Nq7cBOxP7lZ2ZvczOxPblb/9ka/cEnI7XVmrW9gTbr7UpJLSVJVm929seD7w6GyN7mZ2Z/crOxNbmb2Jzerqtq80a9dcmnlVpJTO45PJnnhWmuq6liSdyf55xsdCgAAgGtbEnJPJjldVXdU1a1J7k1yedeay0l+avvxR5P8RXe/4YwcAAAAb92+l1Zu/87b/UkeS3JLks9199NV9WCSze6+nOR3kvx+VV3J6kzcvQu+96W3MDccJHuTm5n9yc3K3uRmZn9ys7rhvVlOnAEAAMyy6IbgAAAA3DyEHAAAwDAHHnJVdbaqnq2qK1X1wB6vf2dV/eH261+qqtsPeiZIFu3Nn6uqZ6rqqar686r6gbdjTo6m/fbnjnUfraquKh+rzaFYsjer6ie2f34+XVV/cNgzcjQt+Hv9+6vq8ar6yvbf7fe8HXNy9FTV56rqxWvdQ7tWfn177z5VVT+65H0PNOSq6pYkDyW5O8mZJBeq6syuZfcleam7fzDJZ5P82kHOBMnivfmVJBvd/SNJvpjk04c7JUfVwv2ZqnpXkp9N8qXDnZCjasnerKrTSX4hyQe7+4eT/JdDH5QjZ+HPzV9K8kh335nVB/P9xuFOyRH2+SRnr/P63UlOb/+5mOQ3l7zpQZ+RuyvJle5+rrtfSfJwkvO71pxP8rvbj7+Y5MNVtdcNxmGd9t2b3f14d39j+/CJrO6hCIdhyc/OJPnVrP6D4V8OcziOtCV78xNJHurul5Kku1885Bk5mpbszU7yPduP35033hcZDkR3/1Wuf4/t80l+r1eeSPKeqvq+/d73oEPutiTP7zje2n5uzzXd/WqSl5N87wHPBUv25k73JfmzA50IvmXf/VlVdyY51d1/epiDceQt+dn53iTvraq/rqonqup6/wsN67Jkb/5Kko9V1VaSR5P8zOGMBvt6s/8uTbLgPnJv0V5n1nbf72DJGli3xfuuqj6WZCPJjx3oRPAt192fVfWOrC5F//hhDQTblvzsPJbV5UEfyupKhv9ZVe/v7v9zwLNxtC3ZmxeSfL67/1tV/Yes7oH8/u7+fwc/HlzXDfXQQZ+R20pyasfxybzxNPbra6rqWFanuq936hHWYcneTFV9JMkvJjnX3d88pNlgv/35riTvT/KXVfX3ST6Q5LIPPOEQLP17/U+6+1+7+++SPJtV2MFBWrI370vySJJ0998k+a4kxw9lOri+Rf8u3e2gQ+7JJKer6o6qujWrXyy9vGvN5SQ/tf34o0n+ot2lnIO3797cvnTtt7OKOL/jwWG67v7s7pe7+3h3397dt2f1O5znunvz7RmXI2TJ3+t/nOTHk6Sqjmd1qeVzhzolR9GSvfm1JB9Okqr6oaxC7uqhTgl7u5zkJ7c/vfIDSV7u7n/c74sO9NLK7n61qu5P8liSW5J8rrufrqoHk2x29+Ukv5PVqe0rWZ2Ju/cgZ4Jk8d78TJLvTvJH25+/87XuPve2Dc2RsXB/wqFbuDcfS/KfquqZJP83yc9399ffvqk5ChbuzU8m+e9V9V+zumzt404ecBiq6gtZXW5+fPt3NH85yXckSXf/Vla/s3lPkitJvpHkpxe9r/0LAAAwy4HfEBwAAID1EnIAAADDCDkAAIBhhBwAAMAwQg4AAGAYIQcAADCMkAMAABjm/wOg8bZL2PjgXgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x864 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#correlation matrix\n",
    "f, ax=plt.subplots(figsize=(15,12))\n",
    "sns.heatmap(dg_train.corr(),vmax=0.8,square=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col1</th>\n",
       "      <th>col2</th>\n",
       "      <th>col3</th>\n",
       "      <th>col4</th>\n",
       "      <th>col5</th>\n",
       "      <th>col6</th>\n",
       "      <th>col7</th>\n",
       "      <th>col8</th>\n",
       "      <th>col9</th>\n",
       "      <th>col10</th>\n",
       "      <th>...</th>\n",
       "      <th>col3796</th>\n",
       "      <th>col3797</th>\n",
       "      <th>col3798</th>\n",
       "      <th>col3799</th>\n",
       "      <th>col3800</th>\n",
       "      <th>col3801</th>\n",
       "      <th>col3802</th>\n",
       "      <th>col3803</th>\n",
       "      <th>col3804</th>\n",
       "      <th>col3805</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>col1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.023021</td>\n",
       "      <td>-0.006246</td>\n",
       "      <td>-0.004640</td>\n",
       "      <td>-0.017404</td>\n",
       "      <td>-0.008443</td>\n",
       "      <td>0.014153</td>\n",
       "      <td>0.007600</td>\n",
       "      <td>-0.011705</td>\n",
       "      <td>-0.009988</td>\n",
       "      <td>...</td>\n",
       "      <td>0.012871</td>\n",
       "      <td>-0.007158</td>\n",
       "      <td>-0.000319</td>\n",
       "      <td>0.008844</td>\n",
       "      <td>-0.015955</td>\n",
       "      <td>-0.015515</td>\n",
       "      <td>-0.002605</td>\n",
       "      <td>0.021469</td>\n",
       "      <td>-0.031495</td>\n",
       "      <td>-0.003515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col2</th>\n",
       "      <td>-0.023021</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.035540</td>\n",
       "      <td>0.012894</td>\n",
       "      <td>0.775195</td>\n",
       "      <td>0.019187</td>\n",
       "      <td>0.023151</td>\n",
       "      <td>-0.034217</td>\n",
       "      <td>0.660207</td>\n",
       "      <td>0.840674</td>\n",
       "      <td>...</td>\n",
       "      <td>0.589537</td>\n",
       "      <td>0.164090</td>\n",
       "      <td>-0.048047</td>\n",
       "      <td>-0.625940</td>\n",
       "      <td>0.515572</td>\n",
       "      <td>0.450629</td>\n",
       "      <td>-0.015315</td>\n",
       "      <td>0.162307</td>\n",
       "      <td>0.008688</td>\n",
       "      <td>-0.092660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3</th>\n",
       "      <td>-0.006246</td>\n",
       "      <td>0.035540</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.021266</td>\n",
       "      <td>0.210898</td>\n",
       "      <td>0.943914</td>\n",
       "      <td>-0.046549</td>\n",
       "      <td>-0.020826</td>\n",
       "      <td>-0.093407</td>\n",
       "      <td>0.016214</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.125353</td>\n",
       "      <td>0.058104</td>\n",
       "      <td>-0.010288</td>\n",
       "      <td>0.031057</td>\n",
       "      <td>0.370413</td>\n",
       "      <td>0.779676</td>\n",
       "      <td>0.039939</td>\n",
       "      <td>0.028412</td>\n",
       "      <td>0.121692</td>\n",
       "      <td>-0.013385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col4</th>\n",
       "      <td>-0.004640</td>\n",
       "      <td>0.012894</td>\n",
       "      <td>0.021266</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.085892</td>\n",
       "      <td>0.032705</td>\n",
       "      <td>-0.035748</td>\n",
       "      <td>0.300494</td>\n",
       "      <td>-0.066170</td>\n",
       "      <td>0.215578</td>\n",
       "      <td>...</td>\n",
       "      <td>0.357673</td>\n",
       "      <td>0.051507</td>\n",
       "      <td>-0.004021</td>\n",
       "      <td>0.043767</td>\n",
       "      <td>0.302934</td>\n",
       "      <td>0.034381</td>\n",
       "      <td>0.125206</td>\n",
       "      <td>-0.084267</td>\n",
       "      <td>0.028033</td>\n",
       "      <td>-0.042158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col5</th>\n",
       "      <td>-0.017404</td>\n",
       "      <td>0.775195</td>\n",
       "      <td>0.210898</td>\n",
       "      <td>0.085892</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.193540</td>\n",
       "      <td>-0.020333</td>\n",
       "      <td>-0.002112</td>\n",
       "      <td>0.582616</td>\n",
       "      <td>0.795382</td>\n",
       "      <td>...</td>\n",
       "      <td>0.523148</td>\n",
       "      <td>0.132278</td>\n",
       "      <td>-0.018699</td>\n",
       "      <td>-0.406621</td>\n",
       "      <td>0.687493</td>\n",
       "      <td>0.553358</td>\n",
       "      <td>0.009481</td>\n",
       "      <td>0.148197</td>\n",
       "      <td>0.097168</td>\n",
       "      <td>-0.021672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col6</th>\n",
       "      <td>-0.008443</td>\n",
       "      <td>0.019187</td>\n",
       "      <td>0.943914</td>\n",
       "      <td>0.032705</td>\n",
       "      <td>0.193540</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.076774</td>\n",
       "      <td>0.012831</td>\n",
       "      <td>-0.088137</td>\n",
       "      <td>-0.003623</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.145976</td>\n",
       "      <td>0.053824</td>\n",
       "      <td>-0.006062</td>\n",
       "      <td>0.043984</td>\n",
       "      <td>0.384498</td>\n",
       "      <td>0.748479</td>\n",
       "      <td>0.037228</td>\n",
       "      <td>-0.023261</td>\n",
       "      <td>0.122911</td>\n",
       "      <td>-0.046773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col7</th>\n",
       "      <td>0.014153</td>\n",
       "      <td>0.023151</td>\n",
       "      <td>-0.046549</td>\n",
       "      <td>-0.035748</td>\n",
       "      <td>-0.020333</td>\n",
       "      <td>-0.076774</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.090596</td>\n",
       "      <td>-0.032582</td>\n",
       "      <td>0.108542</td>\n",
       "      <td>...</td>\n",
       "      <td>0.179871</td>\n",
       "      <td>0.005880</td>\n",
       "      <td>-0.026685</td>\n",
       "      <td>-0.092885</td>\n",
       "      <td>-0.027278</td>\n",
       "      <td>-0.007111</td>\n",
       "      <td>0.140444</td>\n",
       "      <td>0.227457</td>\n",
       "      <td>-0.062465</td>\n",
       "      <td>0.435659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col8</th>\n",
       "      <td>0.007600</td>\n",
       "      <td>-0.034217</td>\n",
       "      <td>-0.020826</td>\n",
       "      <td>0.300494</td>\n",
       "      <td>-0.002112</td>\n",
       "      <td>0.012831</td>\n",
       "      <td>0.090596</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.067069</td>\n",
       "      <td>0.125089</td>\n",
       "      <td>...</td>\n",
       "      <td>0.334160</td>\n",
       "      <td>0.079050</td>\n",
       "      <td>-0.005257</td>\n",
       "      <td>0.090183</td>\n",
       "      <td>0.156113</td>\n",
       "      <td>-0.061899</td>\n",
       "      <td>0.029947</td>\n",
       "      <td>-0.051484</td>\n",
       "      <td>-0.111428</td>\n",
       "      <td>0.116928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col9</th>\n",
       "      <td>-0.011705</td>\n",
       "      <td>0.660207</td>\n",
       "      <td>-0.093407</td>\n",
       "      <td>-0.066170</td>\n",
       "      <td>0.582616</td>\n",
       "      <td>-0.088137</td>\n",
       "      <td>-0.032582</td>\n",
       "      <td>-0.067069</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.630824</td>\n",
       "      <td>...</td>\n",
       "      <td>0.426640</td>\n",
       "      <td>0.047773</td>\n",
       "      <td>-0.019866</td>\n",
       "      <td>-0.397948</td>\n",
       "      <td>0.339090</td>\n",
       "      <td>0.270029</td>\n",
       "      <td>-0.053182</td>\n",
       "      <td>0.126070</td>\n",
       "      <td>0.140007</td>\n",
       "      <td>-0.082167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col10</th>\n",
       "      <td>-0.009988</td>\n",
       "      <td>0.840674</td>\n",
       "      <td>0.016214</td>\n",
       "      <td>0.215578</td>\n",
       "      <td>0.795382</td>\n",
       "      <td>-0.003623</td>\n",
       "      <td>0.108542</td>\n",
       "      <td>0.125089</td>\n",
       "      <td>0.630824</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.762022</td>\n",
       "      <td>0.096946</td>\n",
       "      <td>-0.048482</td>\n",
       "      <td>-0.479075</td>\n",
       "      <td>0.635402</td>\n",
       "      <td>0.448646</td>\n",
       "      <td>0.051324</td>\n",
       "      <td>0.150565</td>\n",
       "      <td>0.073013</td>\n",
       "      <td>0.023081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col11</th>\n",
       "      <td>-0.000865</td>\n",
       "      <td>0.003216</td>\n",
       "      <td>-0.008877</td>\n",
       "      <td>-0.010064</td>\n",
       "      <td>-0.009708</td>\n",
       "      <td>-0.007708</td>\n",
       "      <td>0.030423</td>\n",
       "      <td>0.033570</td>\n",
       "      <td>0.001453</td>\n",
       "      <td>-0.005375</td>\n",
       "      <td>...</td>\n",
       "      <td>0.012460</td>\n",
       "      <td>-0.013279</td>\n",
       "      <td>-0.000592</td>\n",
       "      <td>-0.015252</td>\n",
       "      <td>-0.017388</td>\n",
       "      <td>-0.014575</td>\n",
       "      <td>-0.004833</td>\n",
       "      <td>-0.000700</td>\n",
       "      <td>-0.057873</td>\n",
       "      <td>-0.003928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col12</th>\n",
       "      <td>-0.003762</td>\n",
       "      <td>-0.094898</td>\n",
       "      <td>-0.042707</td>\n",
       "      <td>-0.003251</td>\n",
       "      <td>-0.089628</td>\n",
       "      <td>-0.032072</td>\n",
       "      <td>0.037096</td>\n",
       "      <td>0.018665</td>\n",
       "      <td>-0.074251</td>\n",
       "      <td>-0.081168</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.029179</td>\n",
       "      <td>0.020918</td>\n",
       "      <td>-0.002575</td>\n",
       "      <td>0.091093</td>\n",
       "      <td>-0.053212</td>\n",
       "      <td>-0.072487</td>\n",
       "      <td>0.016770</td>\n",
       "      <td>-0.053769</td>\n",
       "      <td>-0.046467</td>\n",
       "      <td>0.024005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col13</th>\n",
       "      <td>-0.000488</td>\n",
       "      <td>-0.017514</td>\n",
       "      <td>-0.001933</td>\n",
       "      <td>-0.004669</td>\n",
       "      <td>-0.004977</td>\n",
       "      <td>-0.003567</td>\n",
       "      <td>0.020744</td>\n",
       "      <td>0.024809</td>\n",
       "      <td>-0.014483</td>\n",
       "      <td>-0.001479</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015154</td>\n",
       "      <td>0.012630</td>\n",
       "      <td>-0.000334</td>\n",
       "      <td>0.027293</td>\n",
       "      <td>0.005006</td>\n",
       "      <td>-0.009665</td>\n",
       "      <td>-0.002725</td>\n",
       "      <td>0.032966</td>\n",
       "      <td>-0.020680</td>\n",
       "      <td>0.028938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col14</th>\n",
       "      <td>-0.007988</td>\n",
       "      <td>0.816486</td>\n",
       "      <td>0.009818</td>\n",
       "      <td>0.202787</td>\n",
       "      <td>0.762643</td>\n",
       "      <td>-0.006048</td>\n",
       "      <td>0.159823</td>\n",
       "      <td>0.122908</td>\n",
       "      <td>0.624820</td>\n",
       "      <td>0.970340</td>\n",
       "      <td>...</td>\n",
       "      <td>0.789309</td>\n",
       "      <td>0.118627</td>\n",
       "      <td>-0.047595</td>\n",
       "      <td>-0.480737</td>\n",
       "      <td>0.600949</td>\n",
       "      <td>0.409678</td>\n",
       "      <td>0.062583</td>\n",
       "      <td>0.121219</td>\n",
       "      <td>0.015334</td>\n",
       "      <td>0.018651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col15</th>\n",
       "      <td>-0.007410</td>\n",
       "      <td>-0.016084</td>\n",
       "      <td>-0.049800</td>\n",
       "      <td>-0.051581</td>\n",
       "      <td>0.025058</td>\n",
       "      <td>-0.068163</td>\n",
       "      <td>-0.017055</td>\n",
       "      <td>-0.077239</td>\n",
       "      <td>0.047729</td>\n",
       "      <td>0.045962</td>\n",
       "      <td>...</td>\n",
       "      <td>0.039264</td>\n",
       "      <td>0.144818</td>\n",
       "      <td>-0.005073</td>\n",
       "      <td>-0.047572</td>\n",
       "      <td>-0.037092</td>\n",
       "      <td>-0.041049</td>\n",
       "      <td>-0.015363</td>\n",
       "      <td>0.007614</td>\n",
       "      <td>0.049952</td>\n",
       "      <td>0.139036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col16</th>\n",
       "      <td>0.000868</td>\n",
       "      <td>-0.214162</td>\n",
       "      <td>0.044398</td>\n",
       "      <td>0.286759</td>\n",
       "      <td>-0.070022</td>\n",
       "      <td>0.255267</td>\n",
       "      <td>-0.044996</td>\n",
       "      <td>0.389314</td>\n",
       "      <td>-0.136679</td>\n",
       "      <td>-0.045882</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047849</td>\n",
       "      <td>0.052642</td>\n",
       "      <td>-0.003815</td>\n",
       "      <td>0.291635</td>\n",
       "      <td>0.162502</td>\n",
       "      <td>-0.032108</td>\n",
       "      <td>0.029483</td>\n",
       "      <td>-0.092383</td>\n",
       "      <td>0.042192</td>\n",
       "      <td>0.030468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col17</th>\n",
       "      <td>-0.000261</td>\n",
       "      <td>0.020006</td>\n",
       "      <td>0.009550</td>\n",
       "      <td>-0.003283</td>\n",
       "      <td>0.028073</td>\n",
       "      <td>0.011808</td>\n",
       "      <td>-0.008685</td>\n",
       "      <td>-0.004071</td>\n",
       "      <td>0.014749</td>\n",
       "      <td>0.035320</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014073</td>\n",
       "      <td>-0.004001</td>\n",
       "      <td>-0.000178</td>\n",
       "      <td>-0.012236</td>\n",
       "      <td>0.020904</td>\n",
       "      <td>0.025975</td>\n",
       "      <td>-0.001456</td>\n",
       "      <td>0.008035</td>\n",
       "      <td>-0.004762</td>\n",
       "      <td>0.006627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col18</th>\n",
       "      <td>0.002279</td>\n",
       "      <td>-0.051106</td>\n",
       "      <td>-0.893109</td>\n",
       "      <td>-0.026483</td>\n",
       "      <td>-0.160674</td>\n",
       "      <td>-0.932682</td>\n",
       "      <td>0.045976</td>\n",
       "      <td>-0.025310</td>\n",
       "      <td>0.074733</td>\n",
       "      <td>-0.021770</td>\n",
       "      <td>...</td>\n",
       "      <td>0.082321</td>\n",
       "      <td>-0.074351</td>\n",
       "      <td>0.002175</td>\n",
       "      <td>-0.014042</td>\n",
       "      <td>-0.337575</td>\n",
       "      <td>-0.685864</td>\n",
       "      <td>-0.052942</td>\n",
       "      <td>-0.003197</td>\n",
       "      <td>-0.043223</td>\n",
       "      <td>0.021056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col19</th>\n",
       "      <td>-0.000184</td>\n",
       "      <td>0.014463</td>\n",
       "      <td>0.002562</td>\n",
       "      <td>-0.002321</td>\n",
       "      <td>0.026650</td>\n",
       "      <td>0.001119</td>\n",
       "      <td>0.008263</td>\n",
       "      <td>-0.005972</td>\n",
       "      <td>0.012254</td>\n",
       "      <td>0.011525</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005010</td>\n",
       "      <td>-0.002829</td>\n",
       "      <td>-0.000126</td>\n",
       "      <td>-0.011840</td>\n",
       "      <td>0.013395</td>\n",
       "      <td>0.008040</td>\n",
       "      <td>-0.001030</td>\n",
       "      <td>0.015687</td>\n",
       "      <td>-0.003367</td>\n",
       "      <td>0.004686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col20</th>\n",
       "      <td>-0.014243</td>\n",
       "      <td>0.845819</td>\n",
       "      <td>0.122459</td>\n",
       "      <td>0.167909</td>\n",
       "      <td>0.784911</td>\n",
       "      <td>0.105001</td>\n",
       "      <td>0.062751</td>\n",
       "      <td>0.077191</td>\n",
       "      <td>0.629673</td>\n",
       "      <td>0.962815</td>\n",
       "      <td>...</td>\n",
       "      <td>0.646091</td>\n",
       "      <td>0.129468</td>\n",
       "      <td>-0.067691</td>\n",
       "      <td>-0.489461</td>\n",
       "      <td>0.631269</td>\n",
       "      <td>0.546756</td>\n",
       "      <td>0.040408</td>\n",
       "      <td>0.184972</td>\n",
       "      <td>0.166241</td>\n",
       "      <td>0.036718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col21</th>\n",
       "      <td>0.017416</td>\n",
       "      <td>-0.885632</td>\n",
       "      <td>-0.056060</td>\n",
       "      <td>-0.067830</td>\n",
       "      <td>-0.769684</td>\n",
       "      <td>-0.041976</td>\n",
       "      <td>-0.016474</td>\n",
       "      <td>-0.035038</td>\n",
       "      <td>-0.741293</td>\n",
       "      <td>-0.851433</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.638849</td>\n",
       "      <td>-0.171470</td>\n",
       "      <td>0.040946</td>\n",
       "      <td>0.541730</td>\n",
       "      <td>-0.549458</td>\n",
       "      <td>-0.462421</td>\n",
       "      <td>-0.002728</td>\n",
       "      <td>-0.266360</td>\n",
       "      <td>-0.038939</td>\n",
       "      <td>0.050254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col22</th>\n",
       "      <td>0.059399</td>\n",
       "      <td>-0.364952</td>\n",
       "      <td>-0.112073</td>\n",
       "      <td>0.113094</td>\n",
       "      <td>-0.227617</td>\n",
       "      <td>-0.123293</td>\n",
       "      <td>0.068387</td>\n",
       "      <td>0.177357</td>\n",
       "      <td>-0.196789</td>\n",
       "      <td>-0.137207</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010343</td>\n",
       "      <td>-0.033033</td>\n",
       "      <td>0.001440</td>\n",
       "      <td>0.219727</td>\n",
       "      <td>-0.092453</td>\n",
       "      <td>-0.225694</td>\n",
       "      <td>0.045483</td>\n",
       "      <td>0.234651</td>\n",
       "      <td>-0.135275</td>\n",
       "      <td>0.129709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col23</th>\n",
       "      <td>0.003734</td>\n",
       "      <td>0.008160</td>\n",
       "      <td>-0.049829</td>\n",
       "      <td>-0.118434</td>\n",
       "      <td>0.006489</td>\n",
       "      <td>-0.069689</td>\n",
       "      <td>0.293294</td>\n",
       "      <td>0.021632</td>\n",
       "      <td>0.051105</td>\n",
       "      <td>0.058963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018516</td>\n",
       "      <td>-0.093122</td>\n",
       "      <td>0.001473</td>\n",
       "      <td>-0.014141</td>\n",
       "      <td>-0.047449</td>\n",
       "      <td>-0.025285</td>\n",
       "      <td>0.007220</td>\n",
       "      <td>0.062291</td>\n",
       "      <td>0.006221</td>\n",
       "      <td>0.324362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col24</th>\n",
       "      <td>-0.014937</td>\n",
       "      <td>0.881845</td>\n",
       "      <td>0.056003</td>\n",
       "      <td>0.027314</td>\n",
       "      <td>0.688506</td>\n",
       "      <td>0.030230</td>\n",
       "      <td>0.174309</td>\n",
       "      <td>-0.009453</td>\n",
       "      <td>0.587973</td>\n",
       "      <td>0.838673</td>\n",
       "      <td>...</td>\n",
       "      <td>0.582129</td>\n",
       "      <td>0.130943</td>\n",
       "      <td>-0.068659</td>\n",
       "      <td>-0.588985</td>\n",
       "      <td>0.465542</td>\n",
       "      <td>0.445637</td>\n",
       "      <td>0.014827</td>\n",
       "      <td>0.149708</td>\n",
       "      <td>0.032739</td>\n",
       "      <td>0.030204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col25</th>\n",
       "      <td>-0.009602</td>\n",
       "      <td>0.805927</td>\n",
       "      <td>0.180792</td>\n",
       "      <td>0.200876</td>\n",
       "      <td>0.746224</td>\n",
       "      <td>0.166457</td>\n",
       "      <td>0.014116</td>\n",
       "      <td>0.095031</td>\n",
       "      <td>0.500108</td>\n",
       "      <td>0.927072</td>\n",
       "      <td>...</td>\n",
       "      <td>0.611512</td>\n",
       "      <td>0.119500</td>\n",
       "      <td>-0.075125</td>\n",
       "      <td>-0.457731</td>\n",
       "      <td>0.626448</td>\n",
       "      <td>0.566539</td>\n",
       "      <td>0.037682</td>\n",
       "      <td>0.088713</td>\n",
       "      <td>0.163512</td>\n",
       "      <td>-0.068545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col26</th>\n",
       "      <td>-0.002360</td>\n",
       "      <td>-0.039929</td>\n",
       "      <td>-0.018337</td>\n",
       "      <td>-0.016441</td>\n",
       "      <td>-0.043885</td>\n",
       "      <td>-0.015618</td>\n",
       "      <td>0.040798</td>\n",
       "      <td>0.043415</td>\n",
       "      <td>-0.017905</td>\n",
       "      <td>-0.033298</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002927</td>\n",
       "      <td>-0.019398</td>\n",
       "      <td>-0.001615</td>\n",
       "      <td>-0.004541</td>\n",
       "      <td>-0.042068</td>\n",
       "      <td>-0.044864</td>\n",
       "      <td>-0.007511</td>\n",
       "      <td>0.032265</td>\n",
       "      <td>-0.089859</td>\n",
       "      <td>-0.026045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col27</th>\n",
       "      <td>-0.019016</td>\n",
       "      <td>0.789967</td>\n",
       "      <td>-0.008047</td>\n",
       "      <td>0.099251</td>\n",
       "      <td>0.848549</td>\n",
       "      <td>-0.027001</td>\n",
       "      <td>0.032535</td>\n",
       "      <td>0.035069</td>\n",
       "      <td>0.589630</td>\n",
       "      <td>0.817933</td>\n",
       "      <td>...</td>\n",
       "      <td>0.612425</td>\n",
       "      <td>0.121222</td>\n",
       "      <td>-0.021500</td>\n",
       "      <td>-0.427796</td>\n",
       "      <td>0.560562</td>\n",
       "      <td>0.324836</td>\n",
       "      <td>0.001562</td>\n",
       "      <td>0.140772</td>\n",
       "      <td>0.032311</td>\n",
       "      <td>0.011301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col28</th>\n",
       "      <td>-0.021875</td>\n",
       "      <td>0.737729</td>\n",
       "      <td>0.058037</td>\n",
       "      <td>0.455147</td>\n",
       "      <td>0.685551</td>\n",
       "      <td>0.052700</td>\n",
       "      <td>0.079791</td>\n",
       "      <td>0.302313</td>\n",
       "      <td>0.369339</td>\n",
       "      <td>0.862638</td>\n",
       "      <td>...</td>\n",
       "      <td>0.757475</td>\n",
       "      <td>0.153570</td>\n",
       "      <td>-0.038876</td>\n",
       "      <td>-0.405542</td>\n",
       "      <td>0.632019</td>\n",
       "      <td>0.382001</td>\n",
       "      <td>0.088735</td>\n",
       "      <td>0.045434</td>\n",
       "      <td>0.048273</td>\n",
       "      <td>-0.027823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col29</th>\n",
       "      <td>0.003894</td>\n",
       "      <td>0.599055</td>\n",
       "      <td>0.010546</td>\n",
       "      <td>0.337226</td>\n",
       "      <td>0.554914</td>\n",
       "      <td>-0.000492</td>\n",
       "      <td>0.181898</td>\n",
       "      <td>0.303354</td>\n",
       "      <td>0.371502</td>\n",
       "      <td>0.756059</td>\n",
       "      <td>...</td>\n",
       "      <td>0.812764</td>\n",
       "      <td>0.104012</td>\n",
       "      <td>-0.034864</td>\n",
       "      <td>-0.333140</td>\n",
       "      <td>0.528123</td>\n",
       "      <td>0.265588</td>\n",
       "      <td>0.116366</td>\n",
       "      <td>0.259844</td>\n",
       "      <td>-0.282464</td>\n",
       "      <td>0.113936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col30</th>\n",
       "      <td>-0.004301</td>\n",
       "      <td>0.691131</td>\n",
       "      <td>-0.034818</td>\n",
       "      <td>0.189391</td>\n",
       "      <td>0.683190</td>\n",
       "      <td>-0.072303</td>\n",
       "      <td>0.336683</td>\n",
       "      <td>0.145655</td>\n",
       "      <td>0.535527</td>\n",
       "      <td>0.925265</td>\n",
       "      <td>...</td>\n",
       "      <td>0.762650</td>\n",
       "      <td>0.095457</td>\n",
       "      <td>-0.042094</td>\n",
       "      <td>-0.441649</td>\n",
       "      <td>0.548848</td>\n",
       "      <td>0.342355</td>\n",
       "      <td>0.115562</td>\n",
       "      <td>0.318471</td>\n",
       "      <td>0.050346</td>\n",
       "      <td>0.282339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3776</th>\n",
       "      <td>-0.010521</td>\n",
       "      <td>0.827145</td>\n",
       "      <td>0.003897</td>\n",
       "      <td>0.134071</td>\n",
       "      <td>0.779921</td>\n",
       "      <td>-0.016835</td>\n",
       "      <td>0.155658</td>\n",
       "      <td>0.062389</td>\n",
       "      <td>0.695914</td>\n",
       "      <td>0.976373</td>\n",
       "      <td>...</td>\n",
       "      <td>0.727587</td>\n",
       "      <td>0.113238</td>\n",
       "      <td>-0.047700</td>\n",
       "      <td>-0.501680</td>\n",
       "      <td>0.588583</td>\n",
       "      <td>0.434467</td>\n",
       "      <td>0.044468</td>\n",
       "      <td>0.165985</td>\n",
       "      <td>0.094902</td>\n",
       "      <td>0.049876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3777</th>\n",
       "      <td>0.009863</td>\n",
       "      <td>-0.127871</td>\n",
       "      <td>-0.037507</td>\n",
       "      <td>0.267451</td>\n",
       "      <td>-0.064684</td>\n",
       "      <td>-0.018850</td>\n",
       "      <td>0.178294</td>\n",
       "      <td>0.328873</td>\n",
       "      <td>-0.091628</td>\n",
       "      <td>0.078414</td>\n",
       "      <td>...</td>\n",
       "      <td>0.380180</td>\n",
       "      <td>0.021774</td>\n",
       "      <td>-0.011324</td>\n",
       "      <td>0.163359</td>\n",
       "      <td>0.049839</td>\n",
       "      <td>-0.128515</td>\n",
       "      <td>0.066070</td>\n",
       "      <td>-0.116951</td>\n",
       "      <td>-0.225914</td>\n",
       "      <td>0.106961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3778</th>\n",
       "      <td>-0.010917</td>\n",
       "      <td>0.181839</td>\n",
       "      <td>-0.035965</td>\n",
       "      <td>0.003711</td>\n",
       "      <td>0.181700</td>\n",
       "      <td>-0.043617</td>\n",
       "      <td>-0.060171</td>\n",
       "      <td>0.024921</td>\n",
       "      <td>0.126534</td>\n",
       "      <td>0.244133</td>\n",
       "      <td>...</td>\n",
       "      <td>0.200715</td>\n",
       "      <td>-0.045218</td>\n",
       "      <td>-0.007473</td>\n",
       "      <td>-0.083337</td>\n",
       "      <td>0.142035</td>\n",
       "      <td>0.064946</td>\n",
       "      <td>0.033310</td>\n",
       "      <td>0.066722</td>\n",
       "      <td>-0.021341</td>\n",
       "      <td>-0.004239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3779</th>\n",
       "      <td>-0.001501</td>\n",
       "      <td>-0.034909</td>\n",
       "      <td>-0.006370</td>\n",
       "      <td>-0.009440</td>\n",
       "      <td>-0.036898</td>\n",
       "      <td>-0.002258</td>\n",
       "      <td>0.062892</td>\n",
       "      <td>0.044445</td>\n",
       "      <td>-0.033278</td>\n",
       "      <td>-0.026813</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010930</td>\n",
       "      <td>-0.016472</td>\n",
       "      <td>-0.001027</td>\n",
       "      <td>-0.002543</td>\n",
       "      <td>-0.031742</td>\n",
       "      <td>-0.034537</td>\n",
       "      <td>-0.008385</td>\n",
       "      <td>-0.013832</td>\n",
       "      <td>-0.096559</td>\n",
       "      <td>-0.020608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3780</th>\n",
       "      <td>-0.014546</td>\n",
       "      <td>0.739242</td>\n",
       "      <td>0.079287</td>\n",
       "      <td>-0.003018</td>\n",
       "      <td>0.847139</td>\n",
       "      <td>0.067253</td>\n",
       "      <td>-0.000427</td>\n",
       "      <td>-0.090938</td>\n",
       "      <td>0.623433</td>\n",
       "      <td>0.793423</td>\n",
       "      <td>...</td>\n",
       "      <td>0.442410</td>\n",
       "      <td>0.100001</td>\n",
       "      <td>-0.023804</td>\n",
       "      <td>-0.416159</td>\n",
       "      <td>0.625296</td>\n",
       "      <td>0.528720</td>\n",
       "      <td>-0.017936</td>\n",
       "      <td>0.162521</td>\n",
       "      <td>0.166144</td>\n",
       "      <td>0.003899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3781</th>\n",
       "      <td>-0.024063</td>\n",
       "      <td>0.902860</td>\n",
       "      <td>0.064363</td>\n",
       "      <td>-0.022034</td>\n",
       "      <td>0.677823</td>\n",
       "      <td>0.039073</td>\n",
       "      <td>0.093989</td>\n",
       "      <td>-0.054397</td>\n",
       "      <td>0.577020</td>\n",
       "      <td>0.762481</td>\n",
       "      <td>...</td>\n",
       "      <td>0.517169</td>\n",
       "      <td>0.123777</td>\n",
       "      <td>-0.054573</td>\n",
       "      <td>-0.600697</td>\n",
       "      <td>0.438867</td>\n",
       "      <td>0.434703</td>\n",
       "      <td>-0.013219</td>\n",
       "      <td>0.111213</td>\n",
       "      <td>-0.008335</td>\n",
       "      <td>-0.061037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3782</th>\n",
       "      <td>0.003516</td>\n",
       "      <td>0.503305</td>\n",
       "      <td>0.010302</td>\n",
       "      <td>0.327559</td>\n",
       "      <td>0.563891</td>\n",
       "      <td>-0.016685</td>\n",
       "      <td>0.281733</td>\n",
       "      <td>0.259087</td>\n",
       "      <td>0.349697</td>\n",
       "      <td>0.863444</td>\n",
       "      <td>...</td>\n",
       "      <td>0.753502</td>\n",
       "      <td>0.037533</td>\n",
       "      <td>-0.049170</td>\n",
       "      <td>-0.290544</td>\n",
       "      <td>0.540171</td>\n",
       "      <td>0.313889</td>\n",
       "      <td>0.123381</td>\n",
       "      <td>0.145582</td>\n",
       "      <td>0.051189</td>\n",
       "      <td>0.189339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3783</th>\n",
       "      <td>-0.006236</td>\n",
       "      <td>-0.093317</td>\n",
       "      <td>-0.000996</td>\n",
       "      <td>-0.005182</td>\n",
       "      <td>-0.067902</td>\n",
       "      <td>-0.036632</td>\n",
       "      <td>0.195959</td>\n",
       "      <td>0.035818</td>\n",
       "      <td>-0.101392</td>\n",
       "      <td>-0.021273</td>\n",
       "      <td>...</td>\n",
       "      <td>0.034736</td>\n",
       "      <td>-0.039567</td>\n",
       "      <td>-0.004269</td>\n",
       "      <td>0.003275</td>\n",
       "      <td>-0.033660</td>\n",
       "      <td>-0.018211</td>\n",
       "      <td>0.152201</td>\n",
       "      <td>0.204033</td>\n",
       "      <td>-0.006442</td>\n",
       "      <td>0.358837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3784</th>\n",
       "      <td>-0.004481</td>\n",
       "      <td>0.428336</td>\n",
       "      <td>-0.320528</td>\n",
       "      <td>0.418886</td>\n",
       "      <td>0.299112</td>\n",
       "      <td>-0.353911</td>\n",
       "      <td>0.067457</td>\n",
       "      <td>0.288548</td>\n",
       "      <td>0.287428</td>\n",
       "      <td>0.603138</td>\n",
       "      <td>...</td>\n",
       "      <td>0.715528</td>\n",
       "      <td>0.042693</td>\n",
       "      <td>-0.023104</td>\n",
       "      <td>-0.233191</td>\n",
       "      <td>0.246533</td>\n",
       "      <td>-0.105143</td>\n",
       "      <td>0.042810</td>\n",
       "      <td>0.040910</td>\n",
       "      <td>-0.056930</td>\n",
       "      <td>0.024810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3785</th>\n",
       "      <td>0.016218</td>\n",
       "      <td>-0.068606</td>\n",
       "      <td>0.063069</td>\n",
       "      <td>0.334639</td>\n",
       "      <td>0.017079</td>\n",
       "      <td>0.067153</td>\n",
       "      <td>0.062471</td>\n",
       "      <td>0.394288</td>\n",
       "      <td>-0.153896</td>\n",
       "      <td>0.093625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.298093</td>\n",
       "      <td>0.012025</td>\n",
       "      <td>0.002848</td>\n",
       "      <td>0.160329</td>\n",
       "      <td>0.183295</td>\n",
       "      <td>0.037165</td>\n",
       "      <td>0.089529</td>\n",
       "      <td>0.213120</td>\n",
       "      <td>-0.095133</td>\n",
       "      <td>0.203027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3786</th>\n",
       "      <td>-0.012910</td>\n",
       "      <td>0.775173</td>\n",
       "      <td>0.016671</td>\n",
       "      <td>0.148811</td>\n",
       "      <td>0.734922</td>\n",
       "      <td>0.001253</td>\n",
       "      <td>0.160999</td>\n",
       "      <td>0.083796</td>\n",
       "      <td>0.637060</td>\n",
       "      <td>0.913578</td>\n",
       "      <td>...</td>\n",
       "      <td>0.679339</td>\n",
       "      <td>0.124987</td>\n",
       "      <td>-0.038879</td>\n",
       "      <td>-0.466427</td>\n",
       "      <td>0.555835</td>\n",
       "      <td>0.417077</td>\n",
       "      <td>0.040473</td>\n",
       "      <td>0.082000</td>\n",
       "      <td>0.117684</td>\n",
       "      <td>-0.013506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3787</th>\n",
       "      <td>-0.005060</td>\n",
       "      <td>-0.013675</td>\n",
       "      <td>-0.089605</td>\n",
       "      <td>-0.031999</td>\n",
       "      <td>-0.004595</td>\n",
       "      <td>-0.096616</td>\n",
       "      <td>-0.045465</td>\n",
       "      <td>-0.003927</td>\n",
       "      <td>0.080879</td>\n",
       "      <td>0.008808</td>\n",
       "      <td>...</td>\n",
       "      <td>0.080328</td>\n",
       "      <td>0.080151</td>\n",
       "      <td>-0.003464</td>\n",
       "      <td>-0.030378</td>\n",
       "      <td>-0.031860</td>\n",
       "      <td>-0.065516</td>\n",
       "      <td>-0.024216</td>\n",
       "      <td>-0.004197</td>\n",
       "      <td>-0.033835</td>\n",
       "      <td>-0.034985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3788</th>\n",
       "      <td>-0.000320</td>\n",
       "      <td>0.516686</td>\n",
       "      <td>-0.285547</td>\n",
       "      <td>-0.014121</td>\n",
       "      <td>0.429422</td>\n",
       "      <td>-0.301765</td>\n",
       "      <td>0.069333</td>\n",
       "      <td>-0.015228</td>\n",
       "      <td>0.522981</td>\n",
       "      <td>0.603704</td>\n",
       "      <td>...</td>\n",
       "      <td>0.483820</td>\n",
       "      <td>0.013078</td>\n",
       "      <td>-0.020884</td>\n",
       "      <td>-0.289452</td>\n",
       "      <td>0.195518</td>\n",
       "      <td>0.000316</td>\n",
       "      <td>-0.037082</td>\n",
       "      <td>0.085183</td>\n",
       "      <td>0.022274</td>\n",
       "      <td>0.010694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3789</th>\n",
       "      <td>0.003576</td>\n",
       "      <td>0.180829</td>\n",
       "      <td>-0.010160</td>\n",
       "      <td>-0.032037</td>\n",
       "      <td>0.225014</td>\n",
       "      <td>0.008477</td>\n",
       "      <td>-0.068614</td>\n",
       "      <td>-0.003978</td>\n",
       "      <td>0.218188</td>\n",
       "      <td>0.193748</td>\n",
       "      <td>...</td>\n",
       "      <td>0.162199</td>\n",
       "      <td>-0.084489</td>\n",
       "      <td>-0.008877</td>\n",
       "      <td>0.230033</td>\n",
       "      <td>0.166182</td>\n",
       "      <td>0.091485</td>\n",
       "      <td>-0.023654</td>\n",
       "      <td>0.011724</td>\n",
       "      <td>-0.028904</td>\n",
       "      <td>-0.043183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3790</th>\n",
       "      <td>0.014396</td>\n",
       "      <td>-0.655968</td>\n",
       "      <td>-0.025250</td>\n",
       "      <td>-0.022676</td>\n",
       "      <td>-0.550492</td>\n",
       "      <td>-0.002040</td>\n",
       "      <td>-0.152072</td>\n",
       "      <td>0.015434</td>\n",
       "      <td>-0.428656</td>\n",
       "      <td>-0.662876</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.477759</td>\n",
       "      <td>-0.078472</td>\n",
       "      <td>0.037335</td>\n",
       "      <td>0.425206</td>\n",
       "      <td>-0.387805</td>\n",
       "      <td>-0.365943</td>\n",
       "      <td>-0.018963</td>\n",
       "      <td>-0.115725</td>\n",
       "      <td>-0.011598</td>\n",
       "      <td>-0.063918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3791</th>\n",
       "      <td>0.009394</td>\n",
       "      <td>-0.097463</td>\n",
       "      <td>-0.021502</td>\n",
       "      <td>-0.028627</td>\n",
       "      <td>-0.069850</td>\n",
       "      <td>-0.018792</td>\n",
       "      <td>-0.060414</td>\n",
       "      <td>-0.066054</td>\n",
       "      <td>-0.089118</td>\n",
       "      <td>-0.081043</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.085230</td>\n",
       "      <td>-0.017912</td>\n",
       "      <td>-0.002960</td>\n",
       "      <td>0.103554</td>\n",
       "      <td>-0.079426</td>\n",
       "      <td>-0.062214</td>\n",
       "      <td>-0.014820</td>\n",
       "      <td>-0.092694</td>\n",
       "      <td>0.037545</td>\n",
       "      <td>-0.040339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3792</th>\n",
       "      <td>-0.019445</td>\n",
       "      <td>0.968590</td>\n",
       "      <td>0.151374</td>\n",
       "      <td>0.063418</td>\n",
       "      <td>0.827279</td>\n",
       "      <td>0.130349</td>\n",
       "      <td>0.074417</td>\n",
       "      <td>0.005621</td>\n",
       "      <td>0.669993</td>\n",
       "      <td>0.903073</td>\n",
       "      <td>...</td>\n",
       "      <td>0.642588</td>\n",
       "      <td>0.160656</td>\n",
       "      <td>-0.057780</td>\n",
       "      <td>-0.588843</td>\n",
       "      <td>0.609038</td>\n",
       "      <td>0.572721</td>\n",
       "      <td>0.016478</td>\n",
       "      <td>0.194889</td>\n",
       "      <td>0.055427</td>\n",
       "      <td>-0.023318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3793</th>\n",
       "      <td>0.000933</td>\n",
       "      <td>0.016570</td>\n",
       "      <td>0.025516</td>\n",
       "      <td>0.657711</td>\n",
       "      <td>0.133456</td>\n",
       "      <td>0.046843</td>\n",
       "      <td>0.069008</td>\n",
       "      <td>0.634781</td>\n",
       "      <td>-0.086556</td>\n",
       "      <td>0.322752</td>\n",
       "      <td>...</td>\n",
       "      <td>0.472410</td>\n",
       "      <td>0.009371</td>\n",
       "      <td>-0.013845</td>\n",
       "      <td>0.116430</td>\n",
       "      <td>0.365804</td>\n",
       "      <td>0.057043</td>\n",
       "      <td>0.107197</td>\n",
       "      <td>0.009904</td>\n",
       "      <td>0.080194</td>\n",
       "      <td>0.099051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3794</th>\n",
       "      <td>0.014026</td>\n",
       "      <td>0.082601</td>\n",
       "      <td>-0.034288</td>\n",
       "      <td>-0.022909</td>\n",
       "      <td>0.108481</td>\n",
       "      <td>-0.068510</td>\n",
       "      <td>0.401949</td>\n",
       "      <td>0.054651</td>\n",
       "      <td>0.094824</td>\n",
       "      <td>0.256754</td>\n",
       "      <td>...</td>\n",
       "      <td>0.229134</td>\n",
       "      <td>0.019493</td>\n",
       "      <td>-0.025835</td>\n",
       "      <td>-0.137921</td>\n",
       "      <td>0.075815</td>\n",
       "      <td>0.042640</td>\n",
       "      <td>0.051352</td>\n",
       "      <td>0.466994</td>\n",
       "      <td>-0.051846</td>\n",
       "      <td>0.502334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3795</th>\n",
       "      <td>-0.016634</td>\n",
       "      <td>0.809200</td>\n",
       "      <td>0.149409</td>\n",
       "      <td>0.183043</td>\n",
       "      <td>0.825111</td>\n",
       "      <td>0.136311</td>\n",
       "      <td>0.044377</td>\n",
       "      <td>0.057119</td>\n",
       "      <td>0.632458</td>\n",
       "      <td>0.943511</td>\n",
       "      <td>...</td>\n",
       "      <td>0.598178</td>\n",
       "      <td>0.110658</td>\n",
       "      <td>-0.033755</td>\n",
       "      <td>-0.456267</td>\n",
       "      <td>0.690362</td>\n",
       "      <td>0.605837</td>\n",
       "      <td>0.038203</td>\n",
       "      <td>0.157176</td>\n",
       "      <td>0.201392</td>\n",
       "      <td>0.004499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3796</th>\n",
       "      <td>0.012871</td>\n",
       "      <td>0.589537</td>\n",
       "      <td>-0.125353</td>\n",
       "      <td>0.357673</td>\n",
       "      <td>0.523148</td>\n",
       "      <td>-0.145976</td>\n",
       "      <td>0.179871</td>\n",
       "      <td>0.334160</td>\n",
       "      <td>0.426640</td>\n",
       "      <td>0.762022</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.050880</td>\n",
       "      <td>-0.036269</td>\n",
       "      <td>-0.303402</td>\n",
       "      <td>0.456764</td>\n",
       "      <td>0.125572</td>\n",
       "      <td>0.070422</td>\n",
       "      <td>0.103491</td>\n",
       "      <td>-0.206495</td>\n",
       "      <td>0.036300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3797</th>\n",
       "      <td>-0.007158</td>\n",
       "      <td>0.164090</td>\n",
       "      <td>0.058104</td>\n",
       "      <td>0.051507</td>\n",
       "      <td>0.132278</td>\n",
       "      <td>0.053824</td>\n",
       "      <td>0.005880</td>\n",
       "      <td>0.079050</td>\n",
       "      <td>0.047773</td>\n",
       "      <td>0.096946</td>\n",
       "      <td>...</td>\n",
       "      <td>0.050880</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.004900</td>\n",
       "      <td>-0.111711</td>\n",
       "      <td>0.097686</td>\n",
       "      <td>0.096413</td>\n",
       "      <td>0.000102</td>\n",
       "      <td>0.160131</td>\n",
       "      <td>0.044501</td>\n",
       "      <td>0.084394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3798</th>\n",
       "      <td>-0.000319</td>\n",
       "      <td>-0.048047</td>\n",
       "      <td>-0.010288</td>\n",
       "      <td>-0.004021</td>\n",
       "      <td>-0.018699</td>\n",
       "      <td>-0.006062</td>\n",
       "      <td>-0.026685</td>\n",
       "      <td>-0.005257</td>\n",
       "      <td>-0.019866</td>\n",
       "      <td>-0.048482</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.036269</td>\n",
       "      <td>-0.004900</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.032360</td>\n",
       "      <td>-0.013423</td>\n",
       "      <td>-0.022301</td>\n",
       "      <td>-0.001784</td>\n",
       "      <td>-0.003783</td>\n",
       "      <td>-0.035790</td>\n",
       "      <td>0.008117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3799</th>\n",
       "      <td>0.008844</td>\n",
       "      <td>-0.625940</td>\n",
       "      <td>0.031057</td>\n",
       "      <td>0.043767</td>\n",
       "      <td>-0.406621</td>\n",
       "      <td>0.043984</td>\n",
       "      <td>-0.092885</td>\n",
       "      <td>0.090183</td>\n",
       "      <td>-0.397948</td>\n",
       "      <td>-0.479075</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.303402</td>\n",
       "      <td>-0.111711</td>\n",
       "      <td>0.032360</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.228618</td>\n",
       "      <td>-0.225353</td>\n",
       "      <td>-0.005418</td>\n",
       "      <td>-0.202012</td>\n",
       "      <td>0.000237</td>\n",
       "      <td>-0.004871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3800</th>\n",
       "      <td>-0.015955</td>\n",
       "      <td>0.515572</td>\n",
       "      <td>0.370413</td>\n",
       "      <td>0.302934</td>\n",
       "      <td>0.687493</td>\n",
       "      <td>0.384498</td>\n",
       "      <td>-0.027278</td>\n",
       "      <td>0.156113</td>\n",
       "      <td>0.339090</td>\n",
       "      <td>0.635402</td>\n",
       "      <td>...</td>\n",
       "      <td>0.456764</td>\n",
       "      <td>0.097686</td>\n",
       "      <td>-0.013423</td>\n",
       "      <td>-0.228618</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.589111</td>\n",
       "      <td>0.068574</td>\n",
       "      <td>0.117374</td>\n",
       "      <td>0.095114</td>\n",
       "      <td>0.002240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3801</th>\n",
       "      <td>-0.015515</td>\n",
       "      <td>0.450629</td>\n",
       "      <td>0.779676</td>\n",
       "      <td>0.034381</td>\n",
       "      <td>0.553358</td>\n",
       "      <td>0.748479</td>\n",
       "      <td>-0.007111</td>\n",
       "      <td>-0.061899</td>\n",
       "      <td>0.270029</td>\n",
       "      <td>0.448646</td>\n",
       "      <td>...</td>\n",
       "      <td>0.125572</td>\n",
       "      <td>0.096413</td>\n",
       "      <td>-0.022301</td>\n",
       "      <td>-0.225353</td>\n",
       "      <td>0.589111</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.024647</td>\n",
       "      <td>0.104036</td>\n",
       "      <td>0.221558</td>\n",
       "      <td>-0.028596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3802</th>\n",
       "      <td>-0.002605</td>\n",
       "      <td>-0.015315</td>\n",
       "      <td>0.039939</td>\n",
       "      <td>0.125206</td>\n",
       "      <td>0.009481</td>\n",
       "      <td>0.037228</td>\n",
       "      <td>0.140444</td>\n",
       "      <td>0.029947</td>\n",
       "      <td>-0.053182</td>\n",
       "      <td>0.051324</td>\n",
       "      <td>...</td>\n",
       "      <td>0.070422</td>\n",
       "      <td>0.000102</td>\n",
       "      <td>-0.001784</td>\n",
       "      <td>-0.005418</td>\n",
       "      <td>0.068574</td>\n",
       "      <td>0.024647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.096061</td>\n",
       "      <td>-0.036436</td>\n",
       "      <td>0.158673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3803</th>\n",
       "      <td>0.021469</td>\n",
       "      <td>0.162307</td>\n",
       "      <td>0.028412</td>\n",
       "      <td>-0.084267</td>\n",
       "      <td>0.148197</td>\n",
       "      <td>-0.023261</td>\n",
       "      <td>0.227457</td>\n",
       "      <td>-0.051484</td>\n",
       "      <td>0.126070</td>\n",
       "      <td>0.150565</td>\n",
       "      <td>...</td>\n",
       "      <td>0.103491</td>\n",
       "      <td>0.160131</td>\n",
       "      <td>-0.003783</td>\n",
       "      <td>-0.202012</td>\n",
       "      <td>0.117374</td>\n",
       "      <td>0.104036</td>\n",
       "      <td>0.096061</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.037558</td>\n",
       "      <td>0.592652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3804</th>\n",
       "      <td>-0.031495</td>\n",
       "      <td>0.008688</td>\n",
       "      <td>0.121692</td>\n",
       "      <td>0.028033</td>\n",
       "      <td>0.097168</td>\n",
       "      <td>0.122911</td>\n",
       "      <td>-0.062465</td>\n",
       "      <td>-0.111428</td>\n",
       "      <td>0.140007</td>\n",
       "      <td>0.073013</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.206495</td>\n",
       "      <td>0.044501</td>\n",
       "      <td>-0.035790</td>\n",
       "      <td>0.000237</td>\n",
       "      <td>0.095114</td>\n",
       "      <td>0.221558</td>\n",
       "      <td>-0.036436</td>\n",
       "      <td>-0.037558</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.042075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col3805</th>\n",
       "      <td>-0.003515</td>\n",
       "      <td>-0.092660</td>\n",
       "      <td>-0.013385</td>\n",
       "      <td>-0.042158</td>\n",
       "      <td>-0.021672</td>\n",
       "      <td>-0.046773</td>\n",
       "      <td>0.435659</td>\n",
       "      <td>0.116928</td>\n",
       "      <td>-0.082167</td>\n",
       "      <td>0.023081</td>\n",
       "      <td>...</td>\n",
       "      <td>0.036300</td>\n",
       "      <td>0.084394</td>\n",
       "      <td>0.008117</td>\n",
       "      <td>-0.004871</td>\n",
       "      <td>0.002240</td>\n",
       "      <td>-0.028596</td>\n",
       "      <td>0.158673</td>\n",
       "      <td>0.592652</td>\n",
       "      <td>0.042075</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3805 rows × 3805 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             col1      col2      col3      col4      col5      col6      col7  \\\n",
       "col1     1.000000 -0.023021 -0.006246 -0.004640 -0.017404 -0.008443  0.014153   \n",
       "col2    -0.023021  1.000000  0.035540  0.012894  0.775195  0.019187  0.023151   \n",
       "col3    -0.006246  0.035540  1.000000  0.021266  0.210898  0.943914 -0.046549   \n",
       "col4    -0.004640  0.012894  0.021266  1.000000  0.085892  0.032705 -0.035748   \n",
       "col5    -0.017404  0.775195  0.210898  0.085892  1.000000  0.193540 -0.020333   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "col3801 -0.015515  0.450629  0.779676  0.034381  0.553358  0.748479 -0.007111   \n",
       "col3802 -0.002605 -0.015315  0.039939  0.125206  0.009481  0.037228  0.140444   \n",
       "col3803  0.021469  0.162307  0.028412 -0.084267  0.148197 -0.023261  0.227457   \n",
       "col3804 -0.031495  0.008688  0.121692  0.028033  0.097168  0.122911 -0.062465   \n",
       "col3805 -0.003515 -0.092660 -0.013385 -0.042158 -0.021672 -0.046773  0.435659   \n",
       "\n",
       "             col8      col9     col10  ...   col3796   col3797   col3798  \\\n",
       "col1     0.007600 -0.011705 -0.009988  ...  0.012871 -0.007158 -0.000319   \n",
       "col2    -0.034217  0.660207  0.840674  ...  0.589537  0.164090 -0.048047   \n",
       "col3    -0.020826 -0.093407  0.016214  ... -0.125353  0.058104 -0.010288   \n",
       "col4     0.300494 -0.066170  0.215578  ...  0.357673  0.051507 -0.004021   \n",
       "col5    -0.002112  0.582616  0.795382  ...  0.523148  0.132278 -0.018699   \n",
       "...           ...       ...       ...  ...       ...       ...       ...   \n",
       "col3801 -0.061899  0.270029  0.448646  ...  0.125572  0.096413 -0.022301   \n",
       "col3802  0.029947 -0.053182  0.051324  ...  0.070422  0.000102 -0.001784   \n",
       "col3803 -0.051484  0.126070  0.150565  ...  0.103491  0.160131 -0.003783   \n",
       "col3804 -0.111428  0.140007  0.073013  ... -0.206495  0.044501 -0.035790   \n",
       "col3805  0.116928 -0.082167  0.023081  ...  0.036300  0.084394  0.008117   \n",
       "\n",
       "          col3799   col3800   col3801   col3802   col3803   col3804   col3805  \n",
       "col1     0.008844 -0.015955 -0.015515 -0.002605  0.021469 -0.031495 -0.003515  \n",
       "col2    -0.625940  0.515572  0.450629 -0.015315  0.162307  0.008688 -0.092660  \n",
       "col3     0.031057  0.370413  0.779676  0.039939  0.028412  0.121692 -0.013385  \n",
       "col4     0.043767  0.302934  0.034381  0.125206 -0.084267  0.028033 -0.042158  \n",
       "col5    -0.406621  0.687493  0.553358  0.009481  0.148197  0.097168 -0.021672  \n",
       "...           ...       ...       ...       ...       ...       ...       ...  \n",
       "col3801 -0.225353  0.589111  1.000000  0.024647  0.104036  0.221558 -0.028596  \n",
       "col3802 -0.005418  0.068574  0.024647  1.000000  0.096061 -0.036436  0.158673  \n",
       "col3803 -0.202012  0.117374  0.104036  0.096061  1.000000 -0.037558  0.592652  \n",
       "col3804  0.000237  0.095114  0.221558 -0.036436 -0.037558  1.000000  0.042075  \n",
       "col3805 -0.004871  0.002240 -0.028596  0.158673  0.592652  0.042075  1.000000  \n",
       "\n",
       "[3805 rows x 3805 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dg_train.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#z-score 标准化\n",
    "scaler=StandardScaler().fit(dg_train)#标准化的mean var\n",
    "print(scaler)\n",
    "print(scaler.mean_)\n",
    "print(scaler.var_)\n",
    "dg_scaled_train=scaler.transform(dg_train)#标准化结果向量\n",
    "print(dg_scaled_train)\n",
    "np.shape(dg_scaled_train)\n",
    "\n",
    "dg_scaled_test=scaler.transform(dg_test)\n",
    "print(dg_scaled_test)\n",
    "np.shape(dg_scaled_test)#相同标准 标准化测试集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 以下开始特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#基模型的特征选择\n",
    "clf_fs = ensemble.GradientBoostingRegressor(random_state=20)\n",
    "clf_fs =clf_fs.fit(dg_scaled_train, score_train)\n",
    "clf_fs.feature_importances_  \n",
    "\n",
    "model_fs = SelectFromModel(clf_fs, prefit=True)\n",
    "X = model_fs.transform(dg_scaled_train)\n",
    "\n",
    "\n",
    "#SelectFromModel(ensemble.GradientBoostingRegressor()).fit_transform(dg_scaled_train, score_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_predict=model_fs.transform(dg_scaled_test)\n",
    "print(dg_scaled_test)\n",
    "np.shape(X_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "np.shape(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#划分数据集\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, score_train, test_size=0.2, random_state=21)\n",
    "X_train.shape,y_train.shape,X_test.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2、学習（train）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVR模型实现\n",
    "model_svc=svm.SVR()\n",
    "model_svc.fit(X_train,y_train)\n",
    "\n",
    "print(model_svc.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_svc.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVR模型调参数：C gamma\n",
    "param_g=[{\"C\": [1e0, 1e1, 1e2, 1e3],\"gamma\": np.logspace(-4, 0, 5)}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf=GridSearchCV(svm.SVR(),param_grid=param_g,cv=3,scoring=score)\n",
    "    clf.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf.best_estimator_)\n",
    "    \n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds=clf.cv_results_['std_test_score']\n",
    "    params = clf.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVR模型实现\n",
    "model_svc_t=clf.best_estimator_\n",
    "model_svc_t.fit(X_train,y_train)\n",
    "\n",
    "print(model_svc_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_svc_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_svc=clf.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_svcRF=RandomForestRegressor(random_state=160)\n",
    "model_svcRF.fit(X_train,y_train)\n",
    "\n",
    "print(model_svcRF.score(X_test,y_test))#0.4697676007383277\n",
    "\n",
    "y_pred = model_svcRF.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_estimators=[{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf=GridSearchCV(RandomForestRegressor(random_state=160),param_grid=N_estimators,cv=3,scoring=score)\n",
    "    clf.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf.best_estimator_)\n",
    "    \n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds=clf.cv_results_['std_test_score']\n",
    "    params = clf.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_svcRF_t=clf.best_estimator_\n",
    "model_svcRF_t.fit(X_train,y_train)\n",
    "\n",
    "print(model_svcRF_t.score(X_test,y_test))#默认0.4697676007383277\n",
    "\n",
    "y_pred = model_svcRF_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "model_svcRF=clf.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_xgbr = xgb.XGBRegressor(random_state=161)\n",
    "model_xgbr.fit(X_train, y_train)\n",
    "\n",
    "print(model_xgbr.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_xgbr.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#调参 XGboost——n_estimators 其他默认\n",
    "cv_params = [{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "\n",
    "scores_GBM=['r2']\n",
    "\n",
    "for score in scores_GBM:\n",
    "    print(score)\n",
    "    model = XGBRegressor(random_state=161)\n",
    "    optimized_GBM=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM.best_estimator_)\n",
    "    \n",
    "    means_GBM = optimized_GBM.cv_results_['mean_test_score']\n",
    "    stds_GBM=optimized_GBM.cv_results_['std_test_score']\n",
    "    params_GBM = optimized_GBM.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM,mean_score_GBM,std_score_GBM) in zip(params_GBM,means_GBM,stds_GBM):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM,std_score_GBM,param_GBM))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#调参 XGboost——maxdepth minchildweight 其他默认\n",
    "cv_params = [{'max_depth': [3, 5,7,9], 'min_child_weight': [1, 3, 5]}]\n",
    "\n",
    "scores_GBM_mami=['r2']\n",
    "\n",
    "for score in scores_GBM_mami:\n",
    "    print(score)\n",
    "    model = optimized_GBM.best_estimator_\n",
    "    optimized_GBM_mami=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM_mami.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM_mami.best_estimator_)\n",
    "    \n",
    "    means_GBM_mami = optimized_GBM_mami.cv_results_['mean_test_score']\n",
    "    stds_GBM_mami=optimized_GBM_mami.cv_results_['std_test_score']\n",
    "    params_GBM_mami = optimized_GBM_mami.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM_mami,mean_score_GBM_mami,std_score_GBM_mami) in zip(params_GBM_mami,means_GBM_mami,stds_GBM_mami):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM_mami,std_score_GBM_mami,param_GBM_mami))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#调参 XGboost——学习率\n",
    "cv_params = [{'learning_rate':  [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]}]\n",
    "\n",
    "scores_GBM_lr=['r2']\n",
    "\n",
    "for score in scores_GBM_lr:\n",
    "    print(score)\n",
    "    model = optimized_GBM_mami.best_estimator_\n",
    "    optimized_GBM_lr=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM_lr.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM_lr.best_estimator_)\n",
    "    \n",
    "    means_GBM_lr = optimized_GBM_lr.cv_results_['mean_test_score']\n",
    "    stds_GBM_lr=optimized_GBM_lr.cv_results_['std_test_score']\n",
    "    params_GBM_lr = optimized_GBM_lr.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM_lr,mean_score_GBM_lr,std_score_GBM_lr) in zip(params_GBM_lr,means_GBM_lr,stds_GBM_lr):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM_lr,std_score_GBM_lr,param_GBM_lr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_xgbr_t = optimized_GBM.best_estimator_\n",
    "model_xgbr_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_xgbr_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_xgbr_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "model_xgbr = optimized_GBM.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4.kNN回归\n",
    "model_k_neighbor = neighbors.KNeighborsRegressor()\n",
    "model_k_neighbor.fit(X_train, y_train)\n",
    "\n",
    "print(model_k_neighbor.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_k_neighbor.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grids=[\n",
    "    {\n",
    "        'weights':['uniform'],\n",
    "        'n_neighbors':[i for i in range(1,11)]\n",
    "    },\n",
    "    {\n",
    "        'weights':['distance'],\n",
    "        'n_neighbors':[i for i in range(1,11)],\n",
    "        'p':[i for i in range(1,6)]\n",
    "    }]\n",
    "scores_KNN=['r2']\n",
    "\n",
    "for score in scores_KNN:\n",
    "    print(score)\n",
    "    model = neighbors.KNeighborsRegressor()\n",
    "    clf_k=GridSearchCV(estimator=model, param_grid=param_grids, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    clf_k.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_k.best_estimator_)\n",
    "    \n",
    "    means_KNN = clf_k.cv_results_['mean_test_score']\n",
    "    stds_KNN=clf_k.cv_results_['std_test_score']\n",
    "    params_KNN = clf_k.cv_results_['params']\n",
    "    \n",
    "    for (param_KNN,mean_score_KNN,std_score_KNN) in zip(params_KNN,means_KNN,stds_KNN):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_KNN,std_score_KNN,param_KNN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_k_neighbor_t = clf_k.best_estimator_\n",
    "model_k_neighbor_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_k_neighbor_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_k_neighbor_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_k_neighbor = clf_k.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GBRT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gradient_boosting_regressor = ensemble.GradientBoostingRegressor(random_state=162)  \n",
    "\n",
    "model_gradient_boosting_regressor.fit(X_train, y_train)\n",
    "\n",
    "print(model_gradient_boosting_regressor.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_gradient_boosting_regressor.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_estimators=[{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf_gb=GridSearchCV(ensemble.GradientBoostingRegressor(random_state=162),param_grid=N_estimators,cv=3,scoring=score)\n",
    "    clf_gb.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_gb.best_estimator_)\n",
    "    \n",
    "    means = clf_gb.cv_results_['mean_test_score']\n",
    "    stds=clf_gb.cv_results_['std_test_score']\n",
    "    params = clf_gb.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gradient_boosting_regressor_t = clf_gb.best_estimator_\n",
    "\n",
    "model_gradient_boosting_regressor_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_gradient_boosting_regressor_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_gradient_boosting_regressor_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_gradient_boosting_regressor = clf_gb.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二阶段stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# '''创建训练的数据集'''\n",
    "#data, target = make_blobs(n_samples=50000, centers=2, random_state=0, cluster_std=0.60)\n",
    " \n",
    "# '''模型融合中使用到的各个单模型'''\n",
    "clfs = [model_svc,\n",
    "        model_svcRF,\n",
    "        model_xgbr,\n",
    "        model_k_neighbor,\n",
    "        model_gradient_boosting_regressor\n",
    "       ]\n",
    " \n",
    "#'''切分一部分数据作为测试集'''\n",
    "X=X#训练集 数据\n",
    "X_predict=X_predict#测试集 数据\n",
    "y=score_train#训练集 分数\n",
    "#y_predict = \n",
    "\n",
    "\n",
    "dataset_blend_train = np.zeros((X.shape[0], len(clfs)))#第一轮 保存各个模型在训练集上的预测结果 训练集合个数×模型数\n",
    "dataset_blend_test = np.zeros((X_predict.shape[0], len(clfs)))#第一轮 保存各个模型在测试集上的预测结果 训练集合个数×模型数\n",
    "\n",
    "#'''5折stacking'''\n",
    "n_folds = 5\n",
    "kf = KFold(n_folds,True,22)\n",
    "skf=list(kf.split(X))#X或者y\n",
    "\n",
    "for j, clf in enumerate(clfs):\n",
    "    #'''依次训练各个单模型'''\n",
    "    print(j, clf)\n",
    "    dataset_blend_test_j = np.zeros((X_predict.shape[0], len(skf)))#存目前这个模型上的测试集结果(之后求平均)\n",
    "    for i, (train, test) in enumerate(skf):\n",
    "        #'''使用第i个部分作为预测，剩余的部分来训练模型，获得其预测的输出作为第i部分的新特征。'''\n",
    "        print(\"Fold\", i)\n",
    "        X_train, y_train, X_test, y_test = X[train], y[train], X[test], y[test]\n",
    "        clf.fit(X_train, y_train)\n",
    "        y_submission = clf.predict(X_test)#1fold的预测结果\n",
    "        y_submission=y_submission.flatten()#2维数组转1维 KNN需要 \n",
    "        #y_submission.reshape(len(y_submission),1)#一维数组转二维 可以不加\n",
    "        \n",
    "        dataset_blend_train[test, j] = y_submission#在模型顺序对应的j位置 存1fold的预测结果\n",
    "        dataset_blend_test_j[:, i] = clf.predict(X_predict).flatten()#存该模型该折下的测试集预测结果\n",
    "        \n",
    "    #'''对于测试集，直接用这k个模型的预测值均值作为新的特征。'''\n",
    "    dataset_blend_test[:, j] = dataset_blend_test_j.mean(1)#测试集结果按行取平均后储存\n",
    "    \n",
    "    \n",
    "    #print(\"val auc Score: %f\" % r2_score(y_predict, dataset_blend_test[:, j]))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导出数据集到本地\n",
    "submission_train_1=pd.DataFrame(dataset_blend_train)\n",
    "#submission_train_1.head()\n",
    "submission_train_1.to_csv('dataset_blend_train.csv',index=False)#第一轮训练后 train集合预测得到的score集合 训练集样本数x3个模型\n",
    "\n",
    "submission_test_1=pd.DataFrame(dataset_blend_test)\n",
    "#submission_test_1.head()\n",
    "submission_test_1.to_csv('dataset_blend_test.csv',index=False)#第一轮训练后 test集合预测得到的score集合 测试机样本数x3个模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3、予測（predict）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第二轮 \n",
    "model_stacking_svr = svm.SVR()\n",
    "\n",
    "model_stacking_svr.fit(dataset_blend_train, y)\n",
    "y_submission = model_stacking_svr.predict(dataset_blend_test)\n",
    "\n",
    "#生成文件\n",
    "submission_df=pd.DataFrame(data={'Id':id_test,'SalePrice':y_submission})\n",
    "submission_df.head()\n",
    "submission_df.to_csv('baseline_st2_test1.csv',header=False,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#调参数\n",
    "X_train, X_test, y_train, y_test = train_test_split(dataset_blend_train, y, test_size=0.2, random_state=123)\n",
    "print(X_train.shape,y_train.shape,X_test.shape,y_test.shape)\n",
    "\n",
    "#SVR模型调参数：C gamma\n",
    "param_g=[{\"C\": [1e0, 1e1, 1e2, 1e3],\"gamma\": np.logspace(-4, 0, 5)}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf_st=GridSearchCV(svm.SVR(),param_grid=param_g,cv=3,scoring=score)\n",
    "    clf_st.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_st.best_estimator_)\n",
    "    \n",
    "    means = clf_st.cv_results_['mean_test_score']\n",
    "    stds=clf_st.cv_results_['std_test_score']\n",
    "    params = clf_st.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_st_best = clf_st.best_estimator_\n",
    "\n",
    "clf_st_best.fit(dataset_blend_train, y)\n",
    "y_submission = clf_st_best.predict(dataset_blend_test)\n",
    "\n",
    "#生成文件\n",
    "submission_df=pd.DataFrame(data={'Id':id_test,'SalePrice':y_submission})\n",
    "submission_df.head()\n",
    "submission_df.to_csv('baseline_st2_test2.csv',header=False,index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}