ver1_template.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2019/09/21 ver.1 模板\n",
    "## 一、文件内容\n",
    "特征选择：GBRT <br />\n",
    "单模型构建：<br />\n",
    " 1：SVR<br />\n",
    " 2：RF<br />\n",
    " 3：XGBoost<br />\n",
    " 4：KNN<br />\n",
    " 5：GBRT<br />\n",
    "Stacking：<br />\n",
    " SVR默认\n",
    "## 二、数据集\n",
    "### 导入数据集变量<br />\n",
    "元数据集 训练集全部：dg_train →训练集 特征数据：dg_train<br />\n",
    "元数据集 测试集全部：dg_test  →测试集 特征数据：dg_test<br />\n",
    "<br />\n",
    "训练集 得分数据：score_train<br />\n",
    "测试集 序号：id_test<br />\n",
    "训练集 序号：id_train\n",
    "<br />\n",
    "### 处理数据集变量<br />\n",
    "标准化处理后的原训练集 特征数据：dg_scaled_train<br />\n",
    "标准化处理后的测试集合 特征数据：dg_scaled_test<br />\n",
    "### 特征选择结果\n",
    "特征选择后的训练集 特征数据：X<br />\n",
    "特征选择后的测试集 特征数据：X_predict<br />\n",
    "### 在X中继续划分数据<br />\n",
    "X_train：0.8比例的原训练集特征 用作训练和验证<br />\n",
    "X_test：0.2比例的原训练集特征 用作测试（模型选择）<br />\n",
    "y_train：0.8比例的原训练集分数 对应X_train<br />\n",
    "y_test：0.2比例的原训练集分数 对应X_test<br />\n",
    "\n",
    "## 三、模型信息\n",
    "\n",
    "\n",
    "变量名|模型名|超参数设置|随机数种子|训练集上预测值\n",
    ":---------:|:------------------:|:-----------------:|:-----------------:|:-------------------------:\n",
    "model_svc|支持向量回归||NA|\n",
    "model_svcRF|随机森林回归||160|\n",
    "model_xgbr|Xgboost回归||161|\n",
    "model_k_neighbor|KNN回归||NA|\n",
    "model_gradient_boosting_regressor|GBRT||162|\n",
    "\n",
    "## 四、随机数信息\n",
    "位置|使用函数|随机数种子\n",
    ":------:|:-------:|:--------:|\n",
    "特征处理|GBRT|20\n",
    "特征处理|train_test_split|21\n",
    "stacking|KFold|22\n",
    "预测|train_test_split|23\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1、前処理（preprocess）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#预设导入\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import joblib\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#机器学习导入\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import svm\n",
    "from sklearn import linear_model\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import explained_variance_score \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBRegressor\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn import ensemble\n",
    "from sklearn import neighbors\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dg_train=pd.read_csv('train.csv')\n",
    "dg_test=pd.read_csv(\"test.csv\")\n",
    "\n",
    "#单独提取ID列和score列\n",
    "id_train=dg_train[\"ID\"].values\n",
    "id_test=dg_test[\"ID\"].values\n",
    "score_train=dg_train[\"Score\"].values\n",
    "\n",
    "del dg_train[\"ID\"]\n",
    "del dg_train[\"Score\"]\n",
    "del dg_test[\"ID\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StandardScaler(copy=True, with_mean=True, with_std=True)\n",
      "[5.82623261e-04 4.17374998e+00 7.18538490e-01 ... 1.87939626e+00\n",
      " 9.97324521e-01 1.03583133e+00]\n",
      "[7.27939627e-04 7.21597472e-02 1.39518638e-02 ... 2.31160294e+00\n",
      " 6.76420780e-07 3.08326932e+00]\n",
      "[[-0.02159435 -0.43089696  0.8928487  ... -0.96316788 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435 -0.23359638  1.78179021 ...  1.38095879 -0.39457952\n",
      "   1.11859545]\n",
      " [-0.02159435 -0.12936211  0.96904369 ... -0.67771587  2.03718504\n",
      "  -0.58990668]\n",
      " ...\n",
      " [-0.02159435  1.25174197  0.83358594 ...  0.55025397  0.82130276\n",
      "  -0.58990668]\n",
      " [-0.02159435  1.17728892 -0.25007609 ... -1.39923858  0.82130276\n",
      "  -0.58990668]\n",
      " [-0.02159435 -0.48301409 -0.45326272 ...  0.18061334 -0.39457952\n",
      "  -0.58990668]]\n",
      "[[-0.02159435 -0.42717431 -0.74957655 ...  0.41476292  0.82130276\n",
      "  -0.58990668]\n",
      " [-0.02159435 -1.50674353  0.78278928 ... -0.40936467 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435  1.90320616  0.7150604  ...  0.67851006  0.82130276\n",
      "  -0.58990668]\n",
      " ...\n",
      " [-0.02159435 -1.04513462  1.66326468 ... -1.17561258 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435  0.41042251 -0.65644935 ... -0.00815331 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435 -0.73987712  0.12243273 ... -0.52512401  0.82130276\n",
      "  -0.58990668]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(13732, 3805)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#z-score 标准化\n",
    "scaler=StandardScaler().fit(dg_train)#标准化的mean var\n",
    "print(scaler)\n",
    "print(scaler.mean_)\n",
    "print(scaler.var_)\n",
    "dg_scaled_train=scaler.transform(dg_train)#标准化结果向量\n",
    "print(dg_scaled_train)\n",
    "np.shape(dg_scaled_train)\n",
    "\n",
    "dg_scaled_test=scaler.transform(dg_test)\n",
    "print(dg_scaled_test)\n",
    "np.shape(dg_scaled_test)#相同标准 标准化测试集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 以下开始特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#基模型的特征选择\n",
    "clf_fs = ensemble.GradientBoostingRegressor(random_state=20)\n",
    "clf_fs =clf_fs.fit(dg_scaled_train, score_train)\n",
    "clf_fs.feature_importances_  \n",
    "\n",
    "model_fs = SelectFromModel(clf_fs, prefit=True)\n",
    "X = model_fs.transform(dg_scaled_train)\n",
    "\n",
    "\n",
    "#SelectFromModel(ensemble.GradientBoostingRegressor()).fit_transform(dg_scaled_train, score_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.02159435 -0.42717431 -0.74957655 ...  0.41476292  0.82130276\n",
      "  -0.58990668]\n",
      " [-0.02159435 -1.50674353  0.78278928 ... -0.40936467 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435  1.90320616  0.7150604  ...  0.67851006  0.82130276\n",
      "  -0.58990668]\n",
      " ...\n",
      " [-0.02159435 -1.04513462  1.66326468 ... -1.17561258 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435  0.41042251 -0.65644935 ... -0.00815331 -0.39457952\n",
      "  -0.58990668]\n",
      " [-0.02159435 -0.73987712  0.12243273 ... -0.52512401  0.82130276\n",
      "  -0.58990668]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(13732, 356)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_predict=model_fs.transform(dg_scaled_test)\n",
    "print(dg_scaled_test)\n",
    "np.shape(X_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13731, 356)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10984, 356), (10984,), (2747, 356), (2747,))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#划分数据集\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, score_train, test_size=0.2, random_state=21)\n",
    "X_train.shape,y_train.shape,X_test.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2、学習（train）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5894213065555851\n",
      "0.5894213065555851\n"
     ]
    }
   ],
   "source": [
    "#SVR模型实现\n",
    "model_svc=svm.SVR()\n",
    "model_svc.fit(X_train,y_train)\n",
    "\n",
    "print(model_svc.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_svc.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "SVR(C=10.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma=0.001,\n",
      "    kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
      "0.381(+/-0.006) for {'C': 1.0, 'gamma': 0.0001}\n",
      "0.499(+/-0.007) for {'C': 1.0, 'gamma': 0.001}\n",
      "0.508(+/-0.006) for {'C': 1.0, 'gamma': 0.01}\n",
      "0.045(+/-0.002) for {'C': 1.0, 'gamma': 0.1}\n",
      "0.011(+/-0.000) for {'C': 1.0, 'gamma': 1.0}\n",
      "0.439(+/-0.007) for {'C': 10.0, 'gamma': 0.0001}\n",
      "0.535(+/-0.007) for {'C': 10.0, 'gamma': 0.001}\n",
      "0.515(+/-0.009) for {'C': 10.0, 'gamma': 0.01}\n",
      "0.052(+/-0.003) for {'C': 10.0, 'gamma': 0.1}\n",
      "0.013(+/-0.001) for {'C': 10.0, 'gamma': 1.0}\n",
      "0.488(+/-0.007) for {'C': 100.0, 'gamma': 0.0001}\n",
      "0.470(+/-0.006) for {'C': 100.0, 'gamma': 0.001}\n",
      "0.516(+/-0.009) for {'C': 100.0, 'gamma': 0.01}\n",
      "0.052(+/-0.003) for {'C': 100.0, 'gamma': 0.1}\n",
      "0.012(+/-0.002) for {'C': 100.0, 'gamma': 1.0}\n",
      "0.490(+/-0.004) for {'C': 1000.0, 'gamma': 0.0001}\n",
      "0.435(+/-0.006) for {'C': 1000.0, 'gamma': 0.001}\n",
      "0.509(+/-0.017) for {'C': 1000.0, 'gamma': 0.01}\n",
      "0.051(+/-0.004) for {'C': 1000.0, 'gamma': 0.1}\n",
      "0.008(+/-0.007) for {'C': 1000.0, 'gamma': 1.0}\n"
     ]
    }
   ],
   "source": [
    "#SVR模型调参数：C gamma\n",
    "param_g=[{\"C\": [1e0, 1e1, 1e2, 1e3],\"gamma\": np.logspace(-4, 0, 5)}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf=GridSearchCV(svm.SVR(),param_grid=param_g,cv=3,scoring=score)\n",
    "    clf.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf.best_estimator_)\n",
    "    \n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds=clf.cv_results_['std_test_score']\n",
    "    params = clf.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5884181686444214\n",
      "0.5884181686444214\n"
     ]
    }
   ],
   "source": [
    "#SVR模型实现\n",
    "model_svc_t=clf.best_estimator_\n",
    "model_svc_t.fit(X_train,y_train)\n",
    "\n",
    "print(model_svc_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_svc_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_svc=clf.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48585311378303303\n",
      "0.48585311378303303\n"
     ]
    }
   ],
   "source": [
    "model_svcRF=RandomForestRegressor(random_state=160)\n",
    "model_svcRF.fit(X_train,y_train)\n",
    "\n",
    "print(model_svcRF.score(X_test,y_test))#0.4697676007383277\n",
    "\n",
    "y_pred = model_svcRF.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "                      max_features='auto', max_leaf_nodes=None,\n",
      "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                      min_samples_leaf=1, min_samples_split=2,\n",
      "                      min_weight_fraction_leaf=0.0, n_estimators=500,\n",
      "                      n_jobs=None, oob_score=False, random_state=160, verbose=0,\n",
      "                      warm_start=False)\n",
      "0.484(+/-0.009) for {'n_estimators': 20}\n",
      "0.501(+/-0.012) for {'n_estimators': 50}\n",
      "0.507(+/-0.012) for {'n_estimators': 100}\n",
      "0.509(+/-0.013) for {'n_estimators': 150}\n",
      "0.510(+/-0.014) for {'n_estimators': 200}\n",
      "0.511(+/-0.012) for {'n_estimators': 300}\n",
      "0.512(+/-0.012) for {'n_estimators': 400}\n",
      "0.513(+/-0.013) for {'n_estimators': 500}\n"
     ]
    }
   ],
   "source": [
    "N_estimators=[{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf=GridSearchCV(RandomForestRegressor(random_state=160),param_grid=N_estimators,cv=3,scoring=score)\n",
    "    clf.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf.best_estimator_)\n",
    "    \n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds=clf.cv_results_['std_test_score']\n",
    "    params = clf.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5486919128328054\n",
      "0.5486919128328054\n"
     ]
    }
   ],
   "source": [
    "model_svcRF_t=clf.best_estimator_\n",
    "model_svcRF_t.fit(X_train,y_train)\n",
    "\n",
    "print(model_svcRF_t.score(X_test,y_test))#默认0.4697676007383277\n",
    "\n",
    "y_pred = model_svcRF_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "model_svcRF=clf.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[04:19:45] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "0.4851841419105461\n",
      "0.4851841419105461\n"
     ]
    }
   ],
   "source": [
    "model_xgbr = xgb.XGBRegressor(random_state=161)\n",
    "model_xgbr.fit(X_train, y_train)\n",
    "\n",
    "print(model_xgbr.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_xgbr.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[04:22:55] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
      "             max_depth=3, min_child_weight=1, missing=None, n_estimators=500,\n",
      "             n_jobs=1, nthread=None, objective='reg:linear', random_state=161,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "             silent=None, subsample=1, verbosity=1)\n",
      "0.295(+/-0.011) for {'n_estimators': 20}\n",
      "0.422(+/-0.012) for {'n_estimators': 50}\n",
      "0.472(+/-0.011) for {'n_estimators': 100}\n",
      "0.495(+/-0.011) for {'n_estimators': 150}\n",
      "0.508(+/-0.012) for {'n_estimators': 200}\n",
      "0.524(+/-0.013) for {'n_estimators': 300}\n",
      "0.533(+/-0.012) for {'n_estimators': 400}\n",
      "0.538(+/-0.012) for {'n_estimators': 500}\n"
     ]
    }
   ],
   "source": [
    "#调参 XGboost——n_estimators 其他默认\n",
    "cv_params = [{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "\n",
    "scores_GBM=['r2']\n",
    "\n",
    "for score in scores_GBM:\n",
    "    print(score)\n",
    "    model = XGBRegressor(random_state=161)\n",
    "    optimized_GBM=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM.best_estimator_)\n",
    "    \n",
    "    means_GBM = optimized_GBM.cv_results_['mean_test_score']\n",
    "    stds_GBM=optimized_GBM.cv_results_['std_test_score']\n",
    "    params_GBM = optimized_GBM.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM,mean_score_GBM,std_score_GBM) in zip(params_GBM,means_GBM,stds_GBM):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM,std_score_GBM,param_GBM))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "Fitting 3 folds for each of 12 candidates, totalling 36 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  36 out of  36 | elapsed: 21.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[04:44:50] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
      "             max_depth=7, min_child_weight=1, missing=None, n_estimators=500,\n",
      "             n_jobs=1, nthread=None, objective='reg:linear', random_state=161,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "             silent=None, subsample=1, verbosity=1)\n",
      "0.538(+/-0.012) for {'max_depth': 3, 'min_child_weight': 1}\n",
      "0.537(+/-0.009) for {'max_depth': 3, 'min_child_weight': 3}\n",
      "0.537(+/-0.011) for {'max_depth': 3, 'min_child_weight': 5}\n",
      "0.558(+/-0.007) for {'max_depth': 5, 'min_child_weight': 1}\n",
      "0.559(+/-0.012) for {'max_depth': 5, 'min_child_weight': 3}\n",
      "0.560(+/-0.011) for {'max_depth': 5, 'min_child_weight': 5}\n",
      "0.561(+/-0.010) for {'max_depth': 7, 'min_child_weight': 1}\n",
      "0.556(+/-0.007) for {'max_depth': 7, 'min_child_weight': 3}\n",
      "0.561(+/-0.006) for {'max_depth': 7, 'min_child_weight': 5}\n",
      "0.547(+/-0.010) for {'max_depth': 9, 'min_child_weight': 1}\n",
      "0.546(+/-0.011) for {'max_depth': 9, 'min_child_weight': 3}\n",
      "0.552(+/-0.009) for {'max_depth': 9, 'min_child_weight': 5}\n"
     ]
    }
   ],
   "source": [
    "#调参 XGboost——maxdepth minchildweight 其他默认\n",
    "cv_params = [{'max_depth': [3, 5,7,9], 'min_child_weight': [1, 3, 5]}]\n",
    "\n",
    "scores_GBM_mami=['r2']\n",
    "\n",
    "for score in scores_GBM_mami:\n",
    "    print(score)\n",
    "    model = optimized_GBM.best_estimator_\n",
    "    optimized_GBM_mami=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM_mami.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM_mami.best_estimator_)\n",
    "    \n",
    "    means_GBM_mami = optimized_GBM_mami.cv_results_['mean_test_score']\n",
    "    stds_GBM_mami=optimized_GBM_mami.cv_results_['std_test_score']\n",
    "    params_GBM_mami = optimized_GBM_mami.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM_mami,mean_score_GBM_mami,std_score_GBM_mami) in zip(params_GBM_mami,means_GBM_mami,stds_GBM_mami):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM_mami,std_score_GBM_mami,param_GBM_mami))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "Fitting 3 folds for each of 6 candidates, totalling 18 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  18 out of  18 | elapsed: 13.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[05:00:28] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
      "             max_depth=7, min_child_weight=1, missing=None, n_estimators=500,\n",
      "             n_jobs=1, nthread=None, objective='reg:linear', random_state=161,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "             silent=None, subsample=1, verbosity=1)\n",
      "-2.447(+/-0.011) for {'learning_rate': 0.0001}\n",
      "-0.769(+/-0.010) for {'learning_rate': 0.001}\n",
      "0.532(+/-0.011) for {'learning_rate': 0.01}\n",
      "0.561(+/-0.010) for {'learning_rate': 0.1}\n",
      "0.539(+/-0.014) for {'learning_rate': 0.2}\n",
      "0.507(+/-0.011) for {'learning_rate': 0.3}\n"
     ]
    }
   ],
   "source": [
    "#调参 XGboost——学习率\n",
    "cv_params = [{'learning_rate':  [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]}]\n",
    "\n",
    "scores_GBM_lr=['r2']\n",
    "\n",
    "for score in scores_GBM_lr:\n",
    "    print(score)\n",
    "    model = optimized_GBM_mami.best_estimator_\n",
    "    optimized_GBM_lr=GridSearchCV(estimator=model, param_grid=cv_params, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    optimized_GBM_lr.fit(X_train,y_train)\n",
    "    \n",
    "    print(optimized_GBM_lr.best_estimator_)\n",
    "    \n",
    "    means_GBM_lr = optimized_GBM_lr.cv_results_['mean_test_score']\n",
    "    stds_GBM_lr=optimized_GBM_lr.cv_results_['std_test_score']\n",
    "    params_GBM_lr = optimized_GBM_lr.cv_results_['params']\n",
    "    \n",
    "    for (param_GBM_lr,mean_score_GBM_lr,std_score_GBM_lr) in zip(params_GBM_lr,means_GBM_lr,stds_GBM_lr):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_GBM_lr,std_score_GBM_lr,param_GBM_lr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[05:02:49] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "0.5671934948663737\n",
      "0.5671934948663737\n"
     ]
    }
   ],
   "source": [
    "model_xgbr_t = optimized_GBM.best_estimator_\n",
    "model_xgbr_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_xgbr_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_xgbr_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "model_xgbr = optimized_GBM.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4610388756489285\n",
      "0.4610388756489285\n"
     ]
    }
   ],
   "source": [
    "# 4.kNN回归\n",
    "model_k_neighbor = neighbors.KNeighborsRegressor()\n",
    "model_k_neighbor.fit(X_train, y_train)\n",
    "\n",
    "print(model_k_neighbor.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_k_neighbor.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "Fitting 3 folds for each of 60 candidates, totalling 180 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 180 out of 180 | elapsed: 133.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNeighborsRegressor(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "                    metric_params=None, n_jobs=None, n_neighbors=8, p=1,\n",
      "                    weights='distance')\n",
      "0.192(+/-0.012) for {'n_neighbors': 1, 'weights': 'uniform'}\n",
      "0.347(+/-0.007) for {'n_neighbors': 2, 'weights': 'uniform'}\n",
      "0.391(+/-0.006) for {'n_neighbors': 3, 'weights': 'uniform'}\n",
      "0.411(+/-0.007) for {'n_neighbors': 4, 'weights': 'uniform'}\n",
      "0.417(+/-0.010) for {'n_neighbors': 5, 'weights': 'uniform'}\n",
      "0.416(+/-0.009) for {'n_neighbors': 6, 'weights': 'uniform'}\n",
      "0.416(+/-0.011) for {'n_neighbors': 7, 'weights': 'uniform'}\n",
      "0.414(+/-0.011) for {'n_neighbors': 8, 'weights': 'uniform'}\n",
      "0.412(+/-0.013) for {'n_neighbors': 9, 'weights': 'uniform'}\n",
      "0.413(+/-0.012) for {'n_neighbors': 10, 'weights': 'uniform'}\n",
      "0.220(+/-0.011) for {'n_neighbors': 1, 'p': 1, 'weights': 'distance'}\n",
      "0.192(+/-0.012) for {'n_neighbors': 1, 'p': 2, 'weights': 'distance'}\n",
      "0.154(+/-0.021) for {'n_neighbors': 1, 'p': 3, 'weights': 'distance'}\n",
      "0.138(+/-0.012) for {'n_neighbors': 1, 'p': 4, 'weights': 'distance'}\n",
      "0.140(+/-0.005) for {'n_neighbors': 1, 'p': 5, 'weights': 'distance'}\n",
      "0.375(+/-0.012) for {'n_neighbors': 2, 'p': 1, 'weights': 'distance'}\n",
      "0.356(+/-0.007) for {'n_neighbors': 2, 'p': 2, 'weights': 'distance'}\n",
      "0.329(+/-0.003) for {'n_neighbors': 2, 'p': 3, 'weights': 'distance'}\n",
      "0.307(+/-0.011) for {'n_neighbors': 2, 'p': 4, 'weights': 'distance'}\n",
      "0.300(+/-0.013) for {'n_neighbors': 2, 'p': 5, 'weights': 'distance'}\n",
      "0.421(+/-0.009) for {'n_neighbors': 3, 'p': 1, 'weights': 'distance'}\n",
      "0.404(+/-0.007) for {'n_neighbors': 3, 'p': 2, 'weights': 'distance'}\n",
      "0.377(+/-0.010) for {'n_neighbors': 3, 'p': 3, 'weights': 'distance'}\n",
      "0.358(+/-0.014) for {'n_neighbors': 3, 'p': 4, 'weights': 'distance'}\n",
      "0.351(+/-0.019) for {'n_neighbors': 3, 'p': 5, 'weights': 'distance'}\n",
      "0.438(+/-0.010) for {'n_neighbors': 4, 'p': 1, 'weights': 'distance'}\n",
      "0.427(+/-0.008) for {'n_neighbors': 4, 'p': 2, 'weights': 'distance'}\n",
      "0.401(+/-0.015) for {'n_neighbors': 4, 'p': 3, 'weights': 'distance'}\n",
      "0.380(+/-0.013) for {'n_neighbors': 4, 'p': 4, 'weights': 'distance'}\n",
      "0.372(+/-0.020) for {'n_neighbors': 4, 'p': 5, 'weights': 'distance'}\n",
      "0.448(+/-0.011) for {'n_neighbors': 5, 'p': 1, 'weights': 'distance'}\n",
      "0.435(+/-0.011) for {'n_neighbors': 5, 'p': 2, 'weights': 'distance'}\n",
      "0.412(+/-0.016) for {'n_neighbors': 5, 'p': 3, 'weights': 'distance'}\n",
      "0.391(+/-0.015) for {'n_neighbors': 5, 'p': 4, 'weights': 'distance'}\n",
      "0.385(+/-0.021) for {'n_neighbors': 5, 'p': 5, 'weights': 'distance'}\n",
      "0.449(+/-0.011) for {'n_neighbors': 6, 'p': 1, 'weights': 'distance'}\n",
      "0.436(+/-0.009) for {'n_neighbors': 6, 'p': 2, 'weights': 'distance'}\n",
      "0.416(+/-0.013) for {'n_neighbors': 6, 'p': 3, 'weights': 'distance'}\n",
      "0.399(+/-0.017) for {'n_neighbors': 6, 'p': 4, 'weights': 'distance'}\n",
      "0.392(+/-0.015) for {'n_neighbors': 6, 'p': 5, 'weights': 'distance'}\n",
      "0.454(+/-0.011) for {'n_neighbors': 7, 'p': 1, 'weights': 'distance'}\n",
      "0.437(+/-0.011) for {'n_neighbors': 7, 'p': 2, 'weights': 'distance'}\n",
      "0.419(+/-0.012) for {'n_neighbors': 7, 'p': 3, 'weights': 'distance'}\n",
      "0.405(+/-0.014) for {'n_neighbors': 7, 'p': 4, 'weights': 'distance'}\n",
      "0.395(+/-0.015) for {'n_neighbors': 7, 'p': 5, 'weights': 'distance'}\n",
      "0.455(+/-0.011) for {'n_neighbors': 8, 'p': 1, 'weights': 'distance'}\n",
      "0.436(+/-0.011) for {'n_neighbors': 8, 'p': 2, 'weights': 'distance'}\n",
      "0.420(+/-0.012) for {'n_neighbors': 8, 'p': 3, 'weights': 'distance'}\n",
      "0.406(+/-0.012) for {'n_neighbors': 8, 'p': 4, 'weights': 'distance'}\n",
      "0.395(+/-0.015) for {'n_neighbors': 8, 'p': 5, 'weights': 'distance'}\n",
      "0.455(+/-0.011) for {'n_neighbors': 9, 'p': 1, 'weights': 'distance'}\n",
      "0.435(+/-0.012) for {'n_neighbors': 9, 'p': 2, 'weights': 'distance'}\n",
      "0.420(+/-0.012) for {'n_neighbors': 9, 'p': 3, 'weights': 'distance'}\n",
      "0.407(+/-0.010) for {'n_neighbors': 9, 'p': 4, 'weights': 'distance'}\n",
      "0.394(+/-0.012) for {'n_neighbors': 9, 'p': 5, 'weights': 'distance'}\n",
      "0.453(+/-0.011) for {'n_neighbors': 10, 'p': 1, 'weights': 'distance'}\n",
      "0.436(+/-0.010) for {'n_neighbors': 10, 'p': 2, 'weights': 'distance'}\n",
      "0.421(+/-0.012) for {'n_neighbors': 10, 'p': 3, 'weights': 'distance'}\n",
      "0.408(+/-0.010) for {'n_neighbors': 10, 'p': 4, 'weights': 'distance'}\n",
      "0.393(+/-0.010) for {'n_neighbors': 10, 'p': 5, 'weights': 'distance'}\n"
     ]
    }
   ],
   "source": [
    "param_grids=[\n",
    "    {\n",
    "        'weights':['uniform'],\n",
    "        'n_neighbors':[i for i in range(1,11)]\n",
    "    },\n",
    "    {\n",
    "        'weights':['distance'],\n",
    "        'n_neighbors':[i for i in range(1,11)],\n",
    "        'p':[i for i in range(1,6)]\n",
    "    }]\n",
    "scores_KNN=['r2']\n",
    "\n",
    "for score in scores_KNN:\n",
    "    print(score)\n",
    "    model = neighbors.KNeighborsRegressor()\n",
    "    clf_k=GridSearchCV(estimator=model, param_grid=param_grids, scoring=score, cv=3, verbose=1, n_jobs=-1)\n",
    "    clf_k.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_k.best_estimator_)\n",
    "    \n",
    "    means_KNN = clf_k.cv_results_['mean_test_score']\n",
    "    stds_KNN=clf_k.cv_results_['std_test_score']\n",
    "    params_KNN = clf_k.cv_results_['params']\n",
    "    \n",
    "    for (param_KNN,mean_score_KNN,std_score_KNN) in zip(params_KNN,means_KNN,stds_KNN):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score_KNN,std_score_KNN,param_KNN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48903204767072717\n",
      "0.48903204767072717\n"
     ]
    }
   ],
   "source": [
    "model_k_neighbor_t = clf_k.best_estimator_\n",
    "model_k_neighbor_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_k_neighbor_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_k_neighbor_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_k_neighbor = clf_k.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GBRT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4878077429565811\n",
      "0.4878077429565811\n"
     ]
    }
   ],
   "source": [
    "model_gradient_boosting_regressor = ensemble.GradientBoostingRegressor(random_state=162)  \n",
    "\n",
    "model_gradient_boosting_regressor.fit(X_train, y_train)\n",
    "\n",
    "print(model_gradient_boosting_regressor.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_gradient_boosting_regressor.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2\n",
      "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
      "                          learning_rate=0.1, loss='ls', max_depth=3,\n",
      "                          max_features=None, max_leaf_nodes=None,\n",
      "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                          min_samples_leaf=1, min_samples_split=2,\n",
      "                          min_weight_fraction_leaf=0.0, n_estimators=500,\n",
      "                          n_iter_no_change=None, presort='auto',\n",
      "                          random_state=162, subsample=1.0, tol=0.0001,\n",
      "                          validation_fraction=0.1, verbose=0, warm_start=False)\n",
      "0.335(+/-0.011) for {'n_estimators': 20}\n",
      "0.423(+/-0.012) for {'n_estimators': 50}\n",
      "0.472(+/-0.010) for {'n_estimators': 100}\n",
      "0.495(+/-0.011) for {'n_estimators': 150}\n",
      "0.507(+/-0.010) for {'n_estimators': 200}\n",
      "0.522(+/-0.009) for {'n_estimators': 300}\n",
      "0.530(+/-0.010) for {'n_estimators': 400}\n",
      "0.535(+/-0.010) for {'n_estimators': 500}\n"
     ]
    }
   ],
   "source": [
    "N_estimators=[{\"n_estimators\":[20,50,100,150,200,300,400,500]}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf_gb=GridSearchCV(ensemble.GradientBoostingRegressor(random_state=162),param_grid=N_estimators,cv=3,scoring=score)\n",
    "    clf_gb.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_gb.best_estimator_)\n",
    "    \n",
    "    means = clf_gb.cv_results_['mean_test_score']\n",
    "    stds=clf_gb.cv_results_['std_test_score']\n",
    "    params = clf_gb.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5685898225956898\n",
      "0.5685898225956898\n"
     ]
    }
   ],
   "source": [
    "model_gradient_boosting_regressor_t = clf_gb.best_estimator_\n",
    "\n",
    "model_gradient_boosting_regressor_t.fit(X_train, y_train)\n",
    "\n",
    "print(model_gradient_boosting_regressor_t.score(X_test,y_test))\n",
    "\n",
    "y_pred = model_gradient_boosting_regressor_t.predict(X_test)\n",
    "print(r2_score(y_test,y_pred, multioutput='variance_weighted'))\n",
    "\n",
    "model_gradient_boosting_regressor = clf_gb.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二阶段stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 SVR(C=10.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma=0.001,\n",
      "    kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
      "Fold 0\n",
      "Fold 1\n",
      "Fold 2\n",
      "Fold 3\n",
      "Fold 4\n",
      "1 RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "                      max_features='auto', max_leaf_nodes=None,\n",
      "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                      min_samples_leaf=1, min_samples_split=2,\n",
      "                      min_weight_fraction_leaf=0.0, n_estimators=500,\n",
      "                      n_jobs=None, oob_score=False, random_state=160, verbose=0,\n",
      "                      warm_start=False)\n",
      "Fold 0\n",
      "Fold 1\n",
      "Fold 2\n",
      "Fold 3\n",
      "Fold 4\n",
      "2 XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
      "             max_depth=3, min_child_weight=1, missing=None, n_estimators=500,\n",
      "             n_jobs=1, nthread=None, objective='reg:linear', random_state=161,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "             silent=None, subsample=1, verbosity=1)\n",
      "Fold 0\n",
      "[09:10:39] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Fold 1\n",
      "[09:11:31] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Fold 2\n",
      "[09:12:25] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Fold 3\n",
      "[09:13:17] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Fold 4\n",
      "[09:14:12] WARNING: d:\\build\\xgboost\\xgboost-0.90.git\\src\\objective\\regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "3 KNeighborsRegressor(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "                    metric_params=None, n_jobs=None, n_neighbors=8, p=1,\n",
      "                    weights='distance')\n",
      "Fold 0\n",
      "Fold 1\n",
      "Fold 2\n",
      "Fold 3\n",
      "Fold 4\n",
      "4 GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
      "                          learning_rate=0.1, loss='ls', max_depth=3,\n",
      "                          max_features=None, max_leaf_nodes=None,\n",
      "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                          min_samples_leaf=1, min_samples_split=2,\n",
      "                          min_weight_fraction_leaf=0.0, n_estimators=500,\n",
      "                          n_iter_no_change=None, presort='auto',\n",
      "                          random_state=162, subsample=1.0, tol=0.0001,\n",
      "                          validation_fraction=0.1, verbose=0, warm_start=False)\n",
      "Fold 0\n",
      "Fold 1\n",
      "Fold 2\n",
      "Fold 3\n",
      "Fold 4\n"
     ]
    }
   ],
   "source": [
    "# '''创建训练的数据集'''\n",
    "#data, target = make_blobs(n_samples=50000, centers=2, random_state=0, cluster_std=0.60)\n",
    " \n",
    "# '''模型融合中使用到的各个单模型'''\n",
    "clfs = [model_svc,\n",
    "        model_svcRF,\n",
    "        model_xgbr,\n",
    "        model_k_neighbor,\n",
    "        model_gradient_boosting_regressor\n",
    "       ]\n",
    " \n",
    "#'''切分一部分数据作为测试集'''\n",
    "X=X#训练集 数据\n",
    "X_predict=X_predict#测试集 数据\n",
    "y=score_train#训练集 分数\n",
    "#y_predict = \n",
    "\n",
    "\n",
    "dataset_blend_train = np.zeros((X.shape[0], len(clfs)))#第一轮 保存各个模型在训练集上的预测结果 训练集合个数×模型数\n",
    "dataset_blend_test = np.zeros((X_predict.shape[0], len(clfs)))#第一轮 保存各个模型在测试集上的预测结果 训练集合个数×模型数\n",
    "\n",
    "#'''5折stacking'''\n",
    "n_folds = 5\n",
    "kf = KFold(n_folds,True,22)\n",
    "skf=list(kf.split(X))#X或者y\n",
    "\n",
    "for j, clf in enumerate(clfs):\n",
    "    #'''依次训练各个单模型'''\n",
    "    print(j, clf)\n",
    "    dataset_blend_test_j = np.zeros((X_predict.shape[0], len(skf)))#存目前这个模型上的测试集结果(之后求平均)\n",
    "    for i, (train, test) in enumerate(skf):\n",
    "        #'''使用第i个部分作为预测，剩余的部分来训练模型，获得其预测的输出作为第i部分的新特征。'''\n",
    "        print(\"Fold\", i)\n",
    "        X_train, y_train, X_test, y_test = X[train], y[train], X[test], y[test]\n",
    "        clf.fit(X_train, y_train)\n",
    "        y_submission = clf.predict(X_test)#1fold的预测结果\n",
    "        y_submission=y_submission.flatten()#2维数组转1维 KNN需要 \n",
    "        #y_submission.reshape(len(y_submission),1)#一维数组转二维 可以不加\n",
    "        \n",
    "        dataset_blend_train[test, j] = y_submission#在模型顺序对应的j位置 存1fold的预测结果\n",
    "        dataset_blend_test_j[:, i] = clf.predict(X_predict).flatten()#存该模型该折下的测试集预测结果\n",
    "        \n",
    "    #'''对于测试集，直接用这k个模型的预测值均值作为新的特征。'''\n",
    "    dataset_blend_test[:, j] = dataset_blend_test_j.mean(1)#测试集结果按行取平均后储存\n",
    "    \n",
    "    \n",
    "    #print(\"val auc Score: %f\" % r2_score(y_predict, dataset_blend_test[:, j]))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导出数据集到本地\n",
    "submission_train_1=pd.DataFrame(dataset_blend_train)\n",
    "#submission_train_1.head()\n",
    "submission_train_1.to_csv('dataset_blend_train.csv',index=False)#第一轮训练后 train集合预测得到的score集合 训练集样本数x3个模型\n",
    "\n",
    "submission_test_1=pd.DataFrame(dataset_blend_test)\n",
    "#submission_test_1.head()\n",
    "submission_test_1.to_csv('dataset_blend_test.csv',index=False)#第一轮训练后 test集合预测得到的score集合 测试机样本数x3个模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3、予測（predict）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第二轮 \n",
    "model_stacking_svr = svm.SVR()\n",
    "\n",
    "model_stacking_svr.fit(dataset_blend_train, y)\n",
    "y_submission = model_stacking_svr.predict(dataset_blend_test)\n",
    "\n",
    "#生成文件\n",
    "submission_df=pd.DataFrame(data={'Id':id_test,'SalePrice':y_submission})\n",
    "submission_df.head()\n",
    "submission_df.to_csv('baseline_st2_test1.csv',header=False,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10984, 5) (10984,) (2747, 5) (2747,)\n",
      "r2\n",
      "SVR(C=10.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma=0.1,\n",
      "    kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
      "0.594(+/-0.005) for {'C': 1.0, 'gamma': 0.0001}\n",
      "0.614(+/-0.006) for {'C': 1.0, 'gamma': 0.001}\n",
      "0.615(+/-0.007) for {'C': 1.0, 'gamma': 0.01}\n",
      "0.616(+/-0.007) for {'C': 1.0, 'gamma': 0.1}\n",
      "0.613(+/-0.007) for {'C': 1.0, 'gamma': 1.0}\n",
      "0.614(+/-0.006) for {'C': 10.0, 'gamma': 0.0001}\n",
      "0.615(+/-0.007) for {'C': 10.0, 'gamma': 0.001}\n",
      "0.615(+/-0.007) for {'C': 10.0, 'gamma': 0.01}\n",
      "0.617(+/-0.007) for {'C': 10.0, 'gamma': 0.1}\n",
      "0.607(+/-0.005) for {'C': 10.0, 'gamma': 1.0}\n",
      "0.615(+/-0.007) for {'C': 100.0, 'gamma': 0.0001}\n",
      "0.615(+/-0.007) for {'C': 100.0, 'gamma': 0.001}\n",
      "0.615(+/-0.007) for {'C': 100.0, 'gamma': 0.01}\n",
      "0.616(+/-0.007) for {'C': 100.0, 'gamma': 0.1}\n",
      "0.576(+/-0.009) for {'C': 100.0, 'gamma': 1.0}\n",
      "0.614(+/-0.007) for {'C': 1000.0, 'gamma': 0.0001}\n",
      "0.615(+/-0.007) for {'C': 1000.0, 'gamma': 0.001}\n",
      "0.616(+/-0.007) for {'C': 1000.0, 'gamma': 0.01}\n",
      "0.613(+/-0.007) for {'C': 1000.0, 'gamma': 0.1}\n",
      "0.462(+/-0.034) for {'C': 1000.0, 'gamma': 1.0}\n"
     ]
    }
   ],
   "source": [
    "#调参数\n",
    "X_train, X_test, y_train, y_test = train_test_split(dataset_blend_train, y, test_size=0.2, random_state=123)\n",
    "print(X_train.shape,y_train.shape,X_test.shape,y_test.shape)\n",
    "\n",
    "#SVR模型调参数：C gamma\n",
    "param_g=[{\"C\": [1e0, 1e1, 1e2, 1e3],\"gamma\": np.logspace(-4, 0, 5)}]\n",
    "scores_g=['r2']\n",
    "for score in scores_g:\n",
    "    print(score)\n",
    "    clf_st=GridSearchCV(svm.SVR(),param_grid=param_g,cv=3,scoring=score)\n",
    "    clf_st.fit(X_train,y_train)\n",
    "    \n",
    "    print(clf_st.best_estimator_)\n",
    "    \n",
    "    means = clf_st.cv_results_['mean_test_score']\n",
    "    stds=clf_st.cv_results_['std_test_score']\n",
    "    params = clf_st.cv_results_['params']\n",
    "    \n",
    "    for (param,mean_score,std_score) in zip(params,means,stds):\n",
    "        print(\"%0.3f(+/-%0.03f) for %r\"%(mean_score,std_score,param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_st_best = clf_st.best_estimator_\n",
    "\n",
    "clf_st_best.fit(dataset_blend_train, y)\n",
    "y_submission = clf_st_best.predict(dataset_blend_test)\n",
    "\n",
    "#生成文件\n",
    "submission_df=pd.DataFrame(data={'Id':id_test,'SalePrice':y_submission})\n",
    "submission_df.head()\n",
    "submission_df.to_csv('baseline_st2_test2.csv',header=False,index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}