DeepOverfitting-LearningModel.ipynb

{
  "cells":[
    {
      "cell_type":"code",
      "source":[
        "from sklearn.ensemble import RandomForestClassifier, BaggingClassifier, ExtraTreesClassifier\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV, KFold\n",
        "from sklearn.metrics import f1_score, confusion_matrix, accuracy_score\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from tqdm import tqdm\n",
        "import pickle\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import numpy as np\n",
        "from sklearn.base import clone"
      ],
      "execution_count":1,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "# Обучение модели"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "## Вспомогательные функции"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "def compute_meta_feature(clf, X_train, X_test, y_train, cv):\n",
        "    \"\"\"\n",
        "    Computes meta-features using the classifier.\n",
        "    \n",
        "    :arg clf: scikit-learn classifier\n",
        "    :args X_train, y_train: training set\n",
        "    :arg X_test: testing set\n",
        "    :arg cv: cross-validation folding\n",
        "    \"\"\"\n",
        "    X_meta_train = np.zeros_like(y_train, dtype=np.float32)\n",
        "    for train_fold_index, predict_fold_index in cv.split(X_train):\n",
        "        X_fold_train, X_fold_predict = X_train[train_fold_index], X_train[predict_fold_index]\n",
        "        y_fold_train = y_train[train_fold_index]\n",
        "        \n",
        "        folded_clf = clone(clf)\n",
        "        folded_clf.fit(X_fold_train, y_fold_train)\n",
        "        X_meta_train[predict_fold_index] = folded_clf.predict_proba(X_fold_predict)[:, 1]\n",
        "    \n",
        "    meta_clf = clone(clf)\n",
        "    meta_clf.fit(X_train, y_train)\n",
        "    \n",
        "    X_meta_test = meta_clf.predict_proba(X_test)[:, 1]\n",
        "    \n",
        "    return X_meta_train, X_meta_test"
      ],
      "execution_count":2,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "def generate_meta_features(classifiers, X_train, X_test, y_train, cv):\n",
        "    \"\"\"\n",
        "    Generates metafeatures using a list of classifiers.\n",
        "    \n",
        "    :arg classifiers: list of scikit-learn classifiers\n",
        "    :args X_train, y_train: training set\n",
        "    :arg X_test: testing set\n",
        "    :arg cv: cross-validation folding\n",
        "    \"\"\"\n",
        "    features = [\n",
        "        compute_meta_feature(clf, X_train, X_test, y_train, cv)\n",
        "        for clf in tqdm(classifiers)\n",
        "    ]\n",
        "    \n",
        "    stacked_features_train = np.vstack([\n",
        "        features_train for features_train, features_test in features\n",
        "    ]).T\n",
        "\n",
        "    stacked_features_test = np.vstack([\n",
        "        features_test for features_train, features_test in features\n",
        "    ]).T\n",
        "    \n",
        "    return stacked_features_train, stacked_features_test"
      ],
      "execution_count":3,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "## Загружаем данные из pickle"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "model_file = open('processed_data.pkl', 'rb')\n",
        "data = pickle.load(model_file)\n",
        "model_file.close()\n",
        "data"
      ],
      "execution_count":4,
      "outputs":[
        {
          "data":{
            "text\/html":[
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "<\/style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th><\/th>\n",
              "      <th>duration<\/th>\n",
              "      <th>rounds<\/th>\n",
              "      <th>f1_age<\/th>\n",
              "      <th>f2_age<\/th>\n",
              "      <th>f1_isHomeCity<\/th>\n",
              "      <th>f1_isHomeCountry<\/th>\n",
              "      <th>f1_isHomeTimezone<\/th>\n",
              "      <th>f2_isHomeCity<\/th>\n",
              "      <th>f2_isHomeCountry<\/th>\n",
              "      <th>f2_isHomeTimezone<\/th>\n",
              "      <th>...<\/th>\n",
              "      <th>hitsTotal_difference<\/th>\n",
              "      <th>takedownTotal_difference<\/th>\n",
              "      <th>submissionAttempts_difference<\/th>\n",
              "      <th>takeovers_difference<\/th>\n",
              "      <th>hitsBodyTotal_difference<\/th>\n",
              "      <th>hitsLegsTotal_difference<\/th>\n",
              "      <th>weight_difference<\/th>\n",
              "      <th>height_difference<\/th>\n",
              "      <th>legSwing_difference<\/th>\n",
              "      <th>eventDate<\/th>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>id<\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "      <th><\/th>\n",
              "    <\/tr>\n",
              "  <\/thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5201<\/th>\n",
              "      <td>104.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>27.0<\/td>\n",
              "      <td>34.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>4.0<\/td>\n",
              "      <td>3.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>-18.6<\/td>\n",
              "      <td>-10.2<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>1993-11-12 00:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>5202<\/th>\n",
              "      <td>52.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>24.0<\/td>\n",
              "      <td>23.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>2.0<\/td>\n",
              "      <td>2.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>2.3<\/td>\n",
              "      <td>-7.6<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>1993-11-12 00:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>5203<\/th>\n",
              "      <td>59.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>23.0<\/td>\n",
              "      <td>34.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>-14.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>-2.0<\/td>\n",
              "      <td>26.8<\/td>\n",
              "      <td>-2.5<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>1993-11-12 00:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>5204<\/th>\n",
              "      <td>57.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>29.0<\/td>\n",
              "      <td>27.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>-12.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>-1.0<\/td>\n",
              "      <td>-1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>13.6<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>1993-11-12 00:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>5205<\/th>\n",
              "      <td>138.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>27.0<\/td>\n",
              "      <td>30.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>7.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>2.0<\/td>\n",
              "      <td>-9.5<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>1993-11-12 00:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>...<\/th>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>...<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>5310<\/th>\n",
              "      <td>260.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>25.0<\/td>\n",
              "      <td>25.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>19.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>7.6<\/td>\n",
              "      <td>5.1<\/td>\n",
              "      <td>2019-12-08 05:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>6338<\/th>\n",
              "      <td>300.0<\/td>\n",
              "      <td>3.0<\/td>\n",
              "      <td>30.0<\/td>\n",
              "      <td>22.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>71.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>-2.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>-5.1<\/td>\n",
              "      <td>-7.6<\/td>\n",
              "      <td>2019-12-08 05:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>5311<\/th>\n",
              "      <td>300.0<\/td>\n",
              "      <td>5.0<\/td>\n",
              "      <td>31.0<\/td>\n",
              "      <td>35.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>6.0<\/td>\n",
              "      <td>2.0<\/td>\n",
              "      <td>-1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>2.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>-2.5<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>2019-12-14 00:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>5312<\/th>\n",
              "      <td>300.0<\/td>\n",
              "      <td>3.0<\/td>\n",
              "      <td>33.0<\/td>\n",
              "      <td>33.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>-9.0<\/td>\n",
              "      <td>1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>-5.1<\/td>\n",
              "      <td>5.2<\/td>\n",
              "      <td>2019-12-14 00:00:00<\/td>\n",
              "    <\/tr>\n",
              "    <tr>\n",
              "      <th>6340<\/th>\n",
              "      <td>295.0<\/td>\n",
              "      <td>2.0<\/td>\n",
              "      <td>36.0<\/td>\n",
              "      <td>38.0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>1<\/td>\n",
              "      <td>0<\/td>\n",
              "      <td>...<\/td>\n",
              "      <td>-12.0<\/td>\n",
              "      <td>-2.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>3.0<\/td>\n",
              "      <td>-1.0<\/td>\n",
              "      <td>0.0<\/td>\n",
              "      <td>5.1<\/td>\n",
              "      <td>5.1<\/td>\n",
              "      <td>2019-12-14 00:00:00<\/td>\n",
              "    <\/tr>\n",
              "  <\/tbody>\n",
              "<\/table>\n",
              "<p>6257 rows × 27 columns<\/p>\n",
              "<\/div>"
            ]
          },
          "metadata":{
            
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "data.info()"
      ],
      "execution_count":5,
      "outputs":[
        {
          "name":"stdout",
          "text":[
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 6257 entries, 5201 to 6340\n",
            "Data columns (total 27 columns):\n",
            " #   Column                          Non-Null Count  Dtype         \n",
            "---  ------                          --------------  -----         \n",
            " 0   duration                        6257 non-null   float64       \n",
            " 1   rounds                          6257 non-null   float64       \n",
            " 2   f1_age                          6257 non-null   float64       \n",
            " 3   f2_age                          6257 non-null   float64       \n",
            " 4   f1_isHomeCity                   6257 non-null   int64         \n",
            " 5   f1_isHomeCountry                6257 non-null   int64         \n",
            " 6   f1_isHomeTimezone               6257 non-null   int64         \n",
            " 7   f2_isHomeCity                   6257 non-null   int64         \n",
            " 8   f2_isHomeCountry                6257 non-null   int64         \n",
            " 9   f2_isHomeTimezone               6257 non-null   int64         \n",
            " 10  winner                          6257 non-null   bool          \n",
            " 11  Knockdown_ratio_difference      6257 non-null   float64       \n",
            " 12  wins_by_knockout_difference     6257 non-null   float64       \n",
            " 13  count_of_fights_difference      6257 non-null   float64       \n",
            " 14  striking_accuracy_difference    6257 non-null   float64       \n",
            " 15  wins_by_submissions_difference  6257 non-null   float64       \n",
            " 16  grappling_accuracy_difference   6257 non-null   float64       \n",
            " 17  hitsTotal_difference            6257 non-null   float64       \n",
            " 18  takedownTotal_difference        6257 non-null   float64       \n",
            " 19  submissionAttempts_difference   6257 non-null   float64       \n",
            " 20  takeovers_difference            6257 non-null   float64       \n",
            " 21  hitsBodyTotal_difference        6257 non-null   float64       \n",
            " 22  hitsLegsTotal_difference        6257 non-null   float64       \n",
            " 23  weight_difference               6257 non-null   float64       \n",
            " 24  height_difference               6257 non-null   float64       \n",
            " 25  legSwing_difference             6257 non-null   float64       \n",
            " 26  eventDate                       6257 non-null   datetime64[ns]\n",
            "dtypes: bool(1), datetime64[ns](1), float64(19), int64(6)\n",
            "memory usage: 1.3 MB\n"
          ],
          "output_type":"stream"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "## Выбираем независимые и целевую переменные"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "### Выбираем период для обучающей и тестовой выборки "
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "def X_y_split(df , feature_names, target_column, basline_column):\n",
        "    \"\"\"\n",
        "  Функция разделяет выборку на трейн , тест и выделяет бейслайн\n",
        "  На входе pandas dataframe и перечень фичей для обучения\/ целевая переменная\/ бейслайн фича,\n",
        "  На выходе train\/test X, Y, baseline dataframes\n",
        "  \"\"\"\n",
        "    X = df[feature_names]\n",
        "    Y= df[target_column]\n",
        "    Y_baseline = df[basline_column]\n",
        "    X_train, X_test, Y_train, Y_test = X.iloc[0:-100,:], X.iloc[-100:], Y.iloc[0:-100], Y.iloc[-100:] #В качестве тестового множества берем последние 100 дней\n",
        "    Y_train_baseline , Y_test_baseline = Y_baseline[0:-100],  Y_baseline[-100:]\n",
        "    return X, Y, Y_baseline, X_train, X_test, Y_train, Y_test,  Y_train_baseline , Y_test_baseline"
      ],
      "execution_count":6,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "X = data.drop(columns=['winner'])\n",
        "y = data[['winner', 'eventDate']]\n",
        "X_train = X[X['eventDate'].dt.year < 2018].drop(columns=['eventDate'])\n",
        "X_test = X[X['eventDate'].dt.year >= 2018].drop(columns=['eventDate'])\n",
        "y_with_date_to_train = y[y['eventDate'].dt.year < 2018]\n",
        "y_with_date_to_test = y[y['eventDate'].dt.year >= 2018]"
      ],
      "execution_count":7,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "y_train = y_with_date_to_train['winner']\n",
        "y_test = y_with_date_to_test['winner']"
      ],
      "execution_count":8,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "## Пробуем случайный лес"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "rfc = RandomForestClassifier()\n",
        "\n",
        "rfc.fit(X_train, y_train.values)\n",
        "\n",
        "y_pred_rfc = rfc.predict(X_test)"
      ],
      "execution_count":9,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "### F1_score метрика"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "f1_score(y_test, y_pred_rfc)"
      ],
      "execution_count":10,
      "outputs":[
        {
          "data":{
            "text\/plain":[
              "0.7773584905660378"
            ]
          },
          "metadata":{
            
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "### Матрица ошибок и метрика accuracy"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "plt.figure(figsize=(10,7))\n",
        "cmd_forest = confusion_matrix(y_test, y_pred_rfc)\n",
        "sns.heatmap(cmd_forest ,annot=True , fmt = 'd')\n",
        "plt.xlabel('Predicted')\n",
        "plt.ylabel('Truth')\n",
        "print('Accuracy RandomForest: ', rfc.score(X_test, y_test))"
      ],
      "execution_count":11,
      "outputs":[
        {
          "name":"stdout",
          "text":[
            "Accuracy RandomForest:  0.7389380530973452\n"
          ],
          "output_type":"stream"
        },
        {
          "data":{
            "image\/png":[
              "iVBORw0KGgoAAAANSUhEUgAAAjQAAAGpCAYAAACam6wDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgJElEQVR4nO3deZRddZXo8e8mCYMmaMJkTBACBCVBjYpxeq00YwBtsBU7LqXRl+7YGlpFUcDWh\/iMLYogPgUNwiONAoYGBFFERJlkSJBJEjpNJAwxkSAzARNStfuPOsRLrLop4Na995zz\/bDOqnt\/Z\/oVLqy99t6\/cyIzkSRJKrONOj0BSZKkF8qARpIklZ4BjSRJKj0DGkmSVHoGNJIkqfSGd3oCA1n8qv1cfiV1wEl\/HtXpKUi1dfLd86Kd93v6T3e17G\/tiC13aOvc12eGRpIklV7XZmgkSdIQ6+3p9AxaxgyNJEkqPTM0kiTVVfZ2egYtY0AjSVJd9VYnoLHkJEmSSs8MjSRJNZWWnCRJUulZcpIkSeoeZmgkSaqrCpWczNBIklRXvT2t2wYhIoZFxM0RcXHxfUxEXBYRdxY\/Rzcce3RELImIxRGx74aubUAjSZLa5RPAHQ3fjwIuz8yJwOXFdyJiEjAdmAxMA06OiGHNLmxAI0lSXWVv67YNiIjxwAHA9xuGDwTmFp\/nAgc1jJ+TmaszcymwBJja7PoGNJIk1VVvb8u2iJgZETc2bDPXu9s3gc8CjdHPNpm5AqD4uXUxPg64r+G4ZcXYgGwKliRJL1hmzgHm9LcvIt4JrMzM30bE7oO4XPR3i2YnGNBIklRTbXyw3tuAv4uI\/YFNgc0j4gfA\/RExNjNXRMRYYGVx\/DJg24bzxwPLm93AkpMkSXXVwpJTM5l5dGaOz8zt6Wv2\/VVmfhC4CDi0OOxQ4MLi80XA9IjYJCImABOB+c3uYYZGkiR1yleBeRExA7gXOBggMxdGxDxgEbAWmJWZTdeGG9BIklRXHXiwXmZeAVxRfH4Q2HOA42YDswd7XQMaSZLqapAPxCsDe2gkSVLpmaGRJKmuKvQuJwMaSZLqagOrk8rEkpMkSSo9MzSSJNWVJSdJklR6lpwkSZK6hxkaSZJqagMP3y0VAxpJkuqqQj00lpwkSVLpmaGRJKmuKtQUbEAjSVJdVajkZEAjSVJd+XJKSZKk7mGGRpKkurLkJEmSSq9CTcGWnCRJUumZoZEkqa4sOUmSpNKz5CRJktQ9zNBIklRXFcrQGNBIklRTVXrbtiUnSZJUemZoJEmqK0tOkiSp9Cq0bNuSkyRJKj0zNJIk1ZUlJ0mSVHqWnCRJkrqHGRpJkurKkpMkSSo9S06SJEndwwyNJEl1ZclJkiSVXoUCGktOkiSp9MzQSJJUVxVqCjagkSSpriw5SZIkdQ8zNJIk1ZUlJ0mSVHqWnCRJkrqHGRpJkurKkpMkSSo9S06SJEndwwyNJEl1VaEMjQGNJEl1ldnpGbSMJSdJkjSkImLTiJgfEbdGxMKIOLYY\/2JE\/CEibim2\/RvOOToilkTE4ojYd0P3MEMjSVJdta\/ktBrYIzOfiIgRwDURcUmx78TMPL7x4IiYBEwHJgMvB34ZETtnZs9ANzCgkSSprtoU0GRmAk8UX0cUW7N614HAOZm5GlgaEUuAqcB1A51gyUmSJL1gETEzIm5s2Gaut39YRNwCrAQuy8wbil2HRcRtEXF6RIwuxsYB9zWcvqwYG5ABjSRJdZW9Ldsyc05m7tawzXnWrTJ7MnMKMB6YGhG7AqcAOwJTgBXAN4rDo7\/ZNvtVDGgkSaqr3t7WbYOUmY8AVwDTMvP+ItDpBU6lr6wEfRmZbRtOGw8sb3ZdAxpJkjSkImKriHhp8XkzYC\/gvyJibMNh7wZuLz5fBEyPiE0iYgIwEZjf7B42BUuSVFftew7NWGBuRAyjL5kyLzMvjogzI2IKfeWku4GP9E0rF0bEPGARsBaY1WyFExjQSJJUX+1b5XQb8Lp+xg9pcs5sYPZg72HJSZIklZ4ZGkmS6sp3OUmSpNLL6gQ0lpwkSVLpmaGRJKmmsrc6b9s2oJEkqa4q1ENjyUmSJJWeGRpJkuqqQk3BBjSSJNVVhXpoLDlJkqTSM0MjSVJdVagp2IBGkqS6MqCRJEml1763bQ85e2gkSVLpmaGRJKmuLDmpzoa\/bEvGHncEw7YcDb3JI\/Mu4ZEzL2SLwz7ASw6eRs9DjwLwpxPnsuqqBQBssvP2bPOlj7PRi18E2cs97\/0EuebpTv4aUil98Gsf5dV7vJ7HH3yUL+97xLrx3Q+dxjv+cRo9PT0s\/NVNXPDVH67bN\/rlW\/CFy07kZ988l1+e+pNOTFvdqkLLtg1o9JxlTw8rjzuV1Yt+T7x4M7Y\/71s8ee3NADw898c8fPp5zz5h2EaM\/fpnWfHZr7N68VI2eukocm1PB2Yuld\/1\/3kFV879OYeeMGvd2M5vmcxr9t6N2fsdwdo1axm5xebPOue9X\/gQi664ud1TldrKgEbPWc8DD9PzwMMA5KqnWP37+xi+zRYDHv\/it72B1YuXsnrxUgB6H3m8LfOUqmjJ\/DsYM36rZ439zQf24dJTLmTtmrUAPPHgY+v2vXafN\/Kne+9nzVOr2zpPlUSFnhQ8ZE3BEfGqiDgyIr4VEScVn3cZqvupM4aP25pNd9mRP9+6GIDRH3gX2194Mi+bfTgbbT4SgI23H0dmMv77X2a78\/4fY2a8t5NTlipn6x3GstPUV\/GZH8\/m8B99ke1esyMAG2+2CXv\/y4H87KRzOzxDda3ebN3WYUMS0ETEkcA5QADzgQXF57Mj4qgm582MiBsj4sYfPXLfUExNLRQv2pRx3\/o8K\/\/9e\/SuepJHzv4pd+39v7n7oFmsfeAhtj7yn\/sOHD6Mzd4wmRVHfI17P3AEI\/d+Ky9685SOzl2qkmHDNuJFm4\/k6wf9G+d\/5UxmfOdwAN55+Pv41Wk\/ZfWTZmdUfUNVcpoBTM7MZ3V9RsQJwELgq\/2dlJlzgDkAi1+1X+fDPQ1s+DDGfevzPPaTX\/PEZdcC0PPgI+t2P3LuJYw\/5VgA1v7xTzy14Hf0PNKXBl915QI2mbQjT15\/S7tnLVXSw398iFsuvQGAe279Pdnby8gxo9h+yk68bv838e6jP8Bmm7+Y7E2eXr2GK\/\/j0g7PWN0iXeW0Qb3Ay4F71hsfW+xTyb3sy59k9e\/v4+EzLlg3Nmyr0et6a0bt9VZW39n3P\/+qa37LmBnvJTbdhHz6aTZ746t5eO4F\/V5X0nN32y8W8Mq37Mqd1y9i6wljGT5iOE889DgnvO+Ydccc8MmDWb3qzwYzerYuKBW1ylAFNJ8ELo+IO4FnakevAHYCDhuie6pNNnv9ZF5y0F6sXryUF13wbaBvifaoA97BprvsAAlP\/+F+\/njMtwDofewJHj7jfLY79yTIZNVVC1h15YJO\/gpSaX34W59g5zdPYuToUcy+7hR+euI8rp33Kw752sf4\/KXHs\/bptcz99Hc6PU2p7SKH6LHHEbERMBUYR1\/\/zDJgQWYOar2uJSepM07686hOT0GqrZPvnhftvN+qL3+wZX9rX\/z5H7R17usbsmXbmdkLXD9U15ckSS9QhUpOvstJkiSVng\/WkySprlzlJEmSSs+SkyRJUvcwQyNJUl1V6F1OBjSSJNWVJSdJkqTuYYZGkqSa8l1OkiSp\/Cw5SZIkdQ8zNJIk1VWFMjQGNJIk1VWFlm1bcpIkSaVnhkaSpLqy5CRJksouKxTQWHKSJEmlZ4ZGkqS6qlCGxoBGkqS6qtCTgi05SZKk0jNDI0lSXVlykiRJpVehgMaSkyRJKj0DGkmSaiozW7Y1ExGbRsT8iLg1IhZGxLHF+JiIuCwi7ix+jm445+iIWBIRiyNi3w39LgY0kiTVVW+2bmtuNbBHZr4WmAJMi4g3A0cBl2fmRODy4jsRMQmYDkwGpgEnR8SwZjcwoJEkSUMq+zxRfB1RbAkcCMwtxucCBxWfDwTOyczVmbkUWAJMbXYPAxpJkuqqhRmaiJgZETc2bDMbbxURwyLiFmAlcFlm3gBsk5krAIqfWxeHjwPuazh9WTE2IFc5SZJUU618l1NmzgHmNNnfA0yJiJcCF0TErk0uF\/1dotn9zdBIkqS2ycxHgCvo6425PyLGAhQ\/VxaHLQO2bThtPLC82XUNaCRJqqs2NQVHxFZFZoaI2AzYC\/gv4CLg0OKwQ4ELi88XAdMjYpOImABMBOY3u4clJ0mS6qp9r3IaC8wtViptBMzLzIsj4jpgXkTMAO4FDgbIzIURMQ9YBKwFZhUlqwEZ0EiSpCGVmbcBr+tn\/EFgzwHOmQ3MHuw9DGgkSaqpVjYFd5oBjSRJdVWhgMamYEmSVHpmaCRJqqv2NQUPOQMaSZJqqko9NJacJElS6ZmhkSSpriw5SZKksrPkJEmS1EXM0EiSVFeWnCRJUtmlAY0kSSq9CgU09tBIkqTSM0MjSVJNWXKSJEnlV6GAxpKTJEkqPTM0kiTVlCUnSZJUelUKaCw5SZKk0jNDI0lSTVUpQ2NAI0lSXWV0egYtY8lJkiSVnhkaSZJqypKTJEkqvey15CRJktQ1zNBIklRTlpwkSVLppaucJEmSuocZGkmSasqSkyRJKj1XOUmSJHURMzSSJNVUZqdn0DoGNJIk1ZQlJ0mSpC5ihkaSpJqqUobGgEaSpJqqUg+NJSdJklR6ZmgkSaopS06SJKn0fJeTJElSFzFDI0lSTfkuJ0mSVHq9lpwkSZK6hxkaSZJqqkpNwQY0kiTVVJWWbVtykiRJpWeGRpKkmvLVB5IkqfSyN1q2NRMR20bEryPijohYGBGfKMa\/GBF\/iIhbim3\/hnOOjoglEbE4Ivbd0O8yqAxNRLwV2L7x+Mz8j8GcK0mSam8t8OnMvCkiRgG\/jYjLin0nZubxjQdHxCRgOjAZeDnwy4jYOTN7BrrBBgOaiDgT2BG4BXjmQgkY0EiSVGLteg5NZq4AVhSfH4+IO4BxTU45EDgnM1cDSyNiCTAVuG6gEwaTodkNmJRZpUqbJElq5bLtiJgJzGwYmpOZc\/o5bnvgdcANwNuAwyLiH4Eb6cviPExfsHN9w2nLaB4ADaqH5nbgZYM4TpIk1VRmzsnM3Rq2\/oKZkcB5wCcz8zHgFPqqQFPoy+B845lD+7tFs\/sPmKGJiJ8UJ48CFkXEfGB1w8T\/rtmFJUlSd2tn7SUiRtAXzPwwM8\/vu3\/e37D\/VODi4usyYNuG08cDy5tdv1nJ6fgm+yRJUsm1q4cmIgI4DbgjM09oGB9b9NcAvJu+qhDARcBZEXECfU3BE4H5ze4xYECTmVcWNzsuM49cb2LHAVc+t19HkiTV1NuAQ4DfRcQtxdjngPdHxBT6KkJ3Ax8ByMyFETEPWETfCqlZzVY4weCagvcGjlxvbL9+xiRJUom0611OmXkN\/ffF\/KzJObOB2YO9R7Memo8CHwN2jIjbGnaNAq4d7A0kSVJ3qtL65WYZmrOAS4B\/B45qGH88Mx8a0llJkiQ9B816aB4FHo2I9UtLIyNiZGbeO5QTm3zXbRs+SFLLPbX86k5PQVKbtKspuB0G00PzU\/qadQLYFJgALKbvccSSJKmk2tVD0w4bDGgy89WN3yPi9RRdyJIkSd1gUC+nbFS8WOqNQzEZSZLUPrUqOUXEpxq+bgS8HnhgyGYkSZLaokKLnAaVoRnV8HktfT015w3NdCRJUrvUJkMTEcOAkZn5mTbNR5Ik6Tlr9mC94Zm5tmgCliRJFVOXVU7z6euXuSUiLgLOBVY9s\/OZN2VKkqRy6u30BFpoMD00Y4AHgT34y\/NoEjCgkSRJXaFZQLN1scLpdv4SyDyjSo3RkiTVUvb7vshyahbQDANG0v\/bMQ1oJEkqud4K\/TVvFtCsyMwvtW0mkiRJz1OzgKY6eShJkvRXeiv0p75ZQLNn22YhSZLarko9NBsNtCMzH2rnRCRJkp6v5\/xySkmSVA11ew6NJEmqoFqUnCRJksrCDI0kSTVlyUmSJJVelQIaS06SJKn0zNBIklRTVWoKNqCRJKmmeqsTz1hykiRJ5WeGRpKkmqrLu5wkSVKFZacn0EKWnCRJUumZoZEkqaaq9BwaAxpJkmqqN6rTQ2PJSZIklZ4ZGkmSaqpKTcEGNJIk1VSVemgsOUmSpNIzQyNJUk1V6dUHBjSSJNVUlZ4UbMlJkiSVnhkaSZJqylVOkiSp9KrUQ2PJSZIklZ4ZGkmSaqpKz6ExoJEkqaaq1ENjyUmSJJWeGRpJkmrKpmBJklR6vS3cmomIbSPi1xFxR0QsjIhPFONjIuKyiLiz+Dm64ZyjI2JJRCyOiH039LsY0EiSpKG2Fvh0Zu4CvBmYFRGTgKOAyzNzInB58Z1i33RgMjANODkihjW7gQGNJEk11a4MTWauyMybis+PA3cA44ADgbnFYXOBg4rPBwLnZObqzFwKLAGmNruHPTSSJNVUdqCHJiK2B14H3ABsk5kroC\/oiYiti8PGAdc3nLasGBuQGRpJkvSCRcTMiLixYZvZzzEjgfOAT2bmY80u189Y01XmZmgkSaqpVj5YLzPnAHMG2h8RI+gLZn6YmecXw\/dHxNgiOzMWWFmMLwO2bTh9PLC82f3N0EiSVFNtXOUUwGnAHZl5QsOui4BDi8+HAhc2jE+PiE0iYgIwEZjf7B5maCRJ0lB7G3AI8LuIuKUY+xzwVWBeRMwA7gUOBsjMhRExD1hE3wqpWZnZ0+wGBjSSJNVUu159kJnX0H9fDMCeA5wzG5g92HsY0EiSVFM+KViSJKmLmKGRJKmmWrnKqdMMaCRJqqkqBTSWnCRJUumZoZEkqabatcqpHQxoJEmqqSqtcjKgkSSppuyhkSRJ6iJmaCRJqil7aCRJUun1ViikseQkSZJKzwyNJEk1VaWmYAMaSZJqqjoFJ0tOkiSpAszQSJJUU5acJElS6VXpScGWnCRJUumZoZEkqaaq9BwaAxpJkmqqOuGMJSdJklQBZmgkSaopVzlJkqTSq1IPjSUnSZJUemZoJEmqqerkZwxoJEmqrSr10FhykiRJpWeGRpKkmqpSU7ABjSRJNVWdcMaSkyRJqgAzNJIk1VSVmoINaCRJqqmsUNHJkpMkSSo9MzSSJNWUJSdJklR6VVq2bclJkiSVnhkaSZJqqjr5GQMaSZJqy5KTJElSFzFDoxdk55135KwfnrLu+w4TXsEXjz2ecS9\/GQe8c2\/WrFnDXXfdw4x\/+hSPPvpYB2cqVUdPTw\/\/MOPjbL3Vlpz89WO59FdXc\/JpP+Cue+7j7FO\/ya677AzAtfNv4pvf\/f88\/fRaRowYzqdnzeBNb5jS2cmrq1RplZMZGr0g\/\/3fv2e3N+7Dbm\/ch6lvmsaTTz7Fjy+8hF9efhWvnbIHr3\/D3tx5510cdeRhnZ6qVBk\/OPdCdtj+Feu+77TDdnzzK1\/gDVN2fdZxo1+6Od8+7otccOYpzP78pzn6S8e3e6rqctnCfzrNgEYts+ce\/4u77rqHe+\/9A5f98ip6enoAuP6Gmxg3bmyHZydVwx9XPsBV187nPe\/ad93Yjtu\/ggnbjf+rY3fZeSe23moLAHaasB2r16xhzZo1bZur1E5tD2gi4sPtvqfa433vO5BzfvTjvxr\/8Iem8\/NLf93+CUkVdNxJ3+NTH5tBxHP7v+\/LrriGXXbekY033niIZqYy6m3h1mmdyNAcO9COiJgZETdGxI29vavaOSe9QCNGjOBd79yH\/zzv4meNH33Ux1m7di1nnXV+h2YmVccVv7mBMaNfyuRXTXxO5y256x5OOPl0\/s9n\/nWIZqayqlLJaUiagiPitoF2AdsMdF5mzgHmAAzfeFzn\/+1o0KZN+1tuvvl3rFz5p3VjhxxyMAfsvxd77\/u+Ds5Mqo6bb1vEFddcz9XXLWD1mqdZtepJjjz2axx3zGcHPOePKx\/gE5\/7v3zlC0fwivEvb+NspfYaqlVO2wD7Ag+vNx7AtUN0T3XQ9H846Fnlpn332Z3PHPEx9tjzPTz11J87NzGpQg7\/6Ic5\/KN9Vfv5N93GGWef1zSYeezxJ\/jYZ47hkx\/5EK9\/zeR2TVMl0g2lolYZqpLTxcDIzLxnve1u4Iohuqc6ZLPNNmWvPd\/OBT++ZN3YSd\/8MqNGjuTnl5zDjQt+wXe+\/dUOzlCqtl9e+Rv2POiD3Hr7HXzsM8cw8\/B\/A+Ds837CfcuW890zzuY9h87iPYfO4sGHH+nsZNVVejNbtnVaZBdMoj+WnKTOeGr51Z2eglRbI7bcIdp5v0O2+\/uW\/a09857zm849Ik4H3gmszMxdi7EvAv8MPFAc9rnM\/Fmx72hgBtADfDwzL212fZdtS5JUU9nCbRDOAKb1M35iZk4ptmeCmUnAdGBycc7JETGs2cUNaCRJqqlesmXbhmTmVcBDg5zagcA5mbk6M5cCS4CpzU4woJEkSZ10WETcFhGnR8ToYmwccF\/DMcuKsQEZ0EiSVFOtfA5N47Pkim3mIKZwCrAjMAVYAXyjGO+vH6dpGsiXU0qSVFOtXLbd+Cy553DO\/c98johT6VslDX0ZmW0bDh0PLG92LTM0kiSpIyKi8UV\/7wZuLz5fBEyPiE0iYgIwEZjf7FpmaCRJqqnBNPO2SkScDewObBkRy4BjgN0jYgp95aS7gY8AZObCiJgHLALWArMys6fZ9Q1oJEmqqXa+gykz39\/P8GlNjp8NzB7s9S05SZKk0jNDI0lSTVXpXU4GNJIk1VS3vv7o+bDkJEmSSs8MjSRJNdXOVU5DzYBGkqSasodGkiSVXjuXbQ81e2gkSVLpmaGRJKmm7KGRJEml57JtSZKkLmKGRpKkmnKVkyRJKj1XOUmSJHURMzSSJNWUq5wkSVLpucpJkiSpi5ihkSSppiw5SZKk0nOVkyRJUhcxQyNJUk31Vqgp2IBGkqSaqk44Y8lJkiRVgBkaSZJqylVOkiSp9KoU0FhykiRJpWeGRpKkmqrSqw8MaCRJqilLTpIkSV3EDI0kSTVVpVcfGNBIklRTVeqhseQkSZJKzwyNJEk1VaWmYAMaSZJqypKTJElSFzFDI0lSTVlykiRJpVelZduWnCRJUumZoZEkqaZ6K9QUbEAjSVJNWXKSJEnqImZoJEmqKUtOkiSp9Cw5SZIkdREzNJIk1ZQlJ0mSVHqWnCRJkrqIAY0kSTXVm9mybUMi4vSIWBkRtzeMjYmIyyLizuLn6IZ9R0fEkohYHBH7buj6BjSSJNVUtvCfQTgDmLbe2FHA5Zk5Ebi8+E5ETAKmA5OLc06OiGHNLm5AI0mShlxmXgU8tN7wgcDc4vNc4KCG8XMyc3VmLgWWAFObXd+mYEmSaiqzt2XXioiZwMyGoTmZOWcDp22TmSv65pIrImLrYnwccH3DccuKsQEZ0EiSVFO9LVzlVAQvGwpgBiv6u0WzEyw5SZKkTrk\/IsYCFD9XFuPLgG0bjhsPLG92IQMaSZJqKjNbtj1PFwGHFp8PBS5sGJ8eEZtExARgIjC\/2YUsOUmSVFOtLDltSEScDewObBkRy4BjgK8C8yJiBnAvcDBAZi6MiHnAImAtMCsze5pe\/wVEVUNq+MbjunNiUsU9tfzqTk9Bqq0RW+7QX+\/IkBk\/ZteW\/a1d9tDtbZ37+szQSJJUU92a1Hg+DGgkSaqpKr2c0qZgSZJUemZoJEmqqSq9bduARpKkmrKHRpIklV47l20PNXtoJElS6ZmhkSSppiw5SZKk0nPZtiRJUhcxQyNJUk1ZcpIkSaXnKidJkqQuYoZGkqSasuQkSZJKz1VOkiRJXcQMjSRJNeXLKSVJUulZcpIkSeoiZmgkSaopVzlJkqTSq1IPjSUnSZJUemZoJEmqKUtOkiSp9KoU0FhykiRJpWeGRpKkmqpOfgaiSukmdY+ImJmZczo9D6lu\/G9PdWXJSUNlZqcnINWU\/+2plgxoJElS6RnQSJKk0jOg0VCxhi91hv\/tqZZsCpYkSaVnhkaSJJWeAY0kSSo9Axq1VERMi4jFEbEkIo7q9HykuoiI0yNiZUTc3um5SJ1gQKOWiYhhwHeA\/YBJwPsjYlJnZyXVxhnAtE5PQuoUAxq10lRgSWbelZlrgHOAAzs8J6kWMvMq4KFOz0PqFAMatdI44L6G78uKMUmShpQBjVop+hnzuQCSpCFnQKNWWgZs2\/B9PLC8Q3ORJNWIAY1aaQEwMSImRMTGwHTgog7PSZJUAwY0apnMXAscBlwK3AHMy8yFnZ2VVA8RcTZwHfDKiFgWETM6PSepnXz1gSRJKj0zNJIkqfQMaCRJUukZ0EiSpNIzoJEkSaVnQCNJkkrPgEYqqYjoiYhbIuL2iDg3Il70Aq51RkS8t\/j8\/WYvFY2I3SPirc\/jHndHxJbPd46S1IwBjVReT2XmlMzcFVgD\/EvjzuLt589ZZv5TZi5qcsjuwHMOaCRpKBnQSNVwNbBTkT35dUScBfwuIoZFxNcjYkFE3BYRHwGIPt+OiEUR8VNg62cuFBFXRMRuxedpEXFTRNwaEZdHxPb0BU6HF9mhv4mIrSLivOIeCyLibcW5W0TELyLi5oj4Hv2\/60uSWmJ4pycg6YWJiOHAfsDPi6GpwK6ZuTQiZgKPZuYbI2IT4DcR8QvgdcArgVcD2wCLgNPXu+5WwKnA24trjcnMhyLiu8ATmXl8cdxZwImZeU1EvIK+J0XvAhwDXJOZX4qIA4CZQ\/ovQlKtGdBI5bVZRNxSfL4aOI2+UtD8zFxajO8DvOaZ\/hjgJcBE4O3A2ZnZAyyPiF\/1c\/03A1c9c63MfGiAeewFTIpYl4DZPCJGFff4++Lcn0bEw8\/v15SkDTOgkcrrqcyc0jhQBBWrGoeAf83MS9c7bn9gQ+89iUEcA32l67dk5lP9zMV3q0hqC3topGq7FPhoRIwAiIidI+LFwFXA9KLHZizwt\/2cex3wjoiYUJw7phh\/HBjVcNwv6HspKcVxU4qPVwEfKMb2A0a36peSpPUZ0EjV9n36+mNuiojbge\/Rl5m9ALgT+B1wCnDl+idm5gP09b2cHxG3Aj8qdv0EePczTcHAx4HdiqbjRfxltdWxwNsj4ib6Sl\/3DtHvKEm+bVuSJJWfGRpJklR6BjSSJKn0DGgkSVLpGdBIkqTSM6CRJEmlZ0AjSZJKz4BGkiSV3v8Aia0azkyAtEIAAAAASUVORK5CYII=\n"
            ]
          },
          "metadata":{
            "image\/png":{
              "width":0,
              "height":0
            }
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "## Стекинг"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "cv = KFold(n_splits=10, shuffle=True, random_state=42)\n",
        "\n",
        "stacked_features_train, stacked_features_test = generate_meta_features([\n",
        "    KNeighborsClassifier(n_jobs=-1),\n",
        "    ExtraTreesClassifier(n_estimators=300, n_jobs=-1),\n",
        "], X_train.values, X_test.values, y_train.values, cv)"
      ],
      "execution_count":12,
      "outputs":[
        {
          "name":"stderr",
          "text":[
            "\r  0%|          | 0\/2 [00:00<?, ?it\/s]\r 50%|█████     | 1\/2 [00:02<00:02,  2.35s\/it]\r100%|██████████| 2\/2 [00:26<00:00, 15.40s\/it]\r100%|██████████| 2\/2 [00:26<00:00, 13.44s\/it]\n"
          ],
          "output_type":"stream"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "clf = LogisticRegression(penalty='none', solver='lbfgs')\n",
        "clf.fit(stacked_features_train, y_train)\n",
        "y_pred_clf = clf.predict(stacked_features_test)"
      ],
      "execution_count":13,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "f1_score(y_test, y_pred_clf)"
      ],
      "execution_count":14,
      "outputs":[
        {
          "data":{
            "text\/plain":[
              "0.7702205882352942"
            ]
          },
          "metadata":{
            
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "accuracy_score(y_test, y_pred_clf)"
      ],
      "execution_count":15,
      "outputs":[
        {
          "data":{
            "text\/plain":[
              "0.7234513274336283"
            ]
          },
          "metadata":{
            
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "stacked_features_train, stacked_features_test = generate_meta_features([\n",
        "    KNeighborsClassifier(n_jobs=-1),\n",
        "    RandomForestClassifier(n_estimators=300, n_jobs=-1),\n",
        "], X_train.values, X_test.values, y_train.values, cv)"
      ],
      "execution_count":16,
      "outputs":[
        {
          "name":"stderr",
          "text":[
            "\r  0%|          | 0\/2 [00:00<?, ?it\/s]\r 50%|█████     | 1\/2 [00:01<00:01,  1.95s\/it]\r100%|██████████| 2\/2 [00:27<00:00, 15.71s\/it]\r100%|██████████| 2\/2 [00:27<00:00, 13.64s\/it]\n"
          ],
          "output_type":"stream"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "clf = LogisticRegression(penalty='none', solver='lbfgs')\n",
        "clf.fit(stacked_features_train, y_train)\n",
        "y_pred_clf = clf.predict(stacked_features_test)"
      ],
      "execution_count":17,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "f1_score(y_test, y_pred_clf)"
      ],
      "execution_count":18,
      "outputs":[
        {
          "data":{
            "text\/plain":[
              "0.779467680608365"
            ]
          },
          "metadata":{
            
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "accuracy_score(y_test, y_pred_clf)"
      ],
      "execution_count":19,
      "outputs":[
        {
          "data":{
            "text\/plain":[
              "0.7433628318584071"
            ]
          },
          "metadata":{
            
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "### Стекинг с нормализацией данных"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "scaler = StandardScaler()\n",
        "cover_X_train = scaler.fit_transform(X_train)\n",
        "cover_X_test = scaler.transform(X_test)"
      ],
      "execution_count":20,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "cv = KFold(n_splits=10, shuffle=True, random_state=42)\n",
        "\n",
        "stacked_features_train, stacked_features_test = generate_meta_features([\n",
        "    KNeighborsClassifier(n_jobs=-1),\n",
        "    ExtraTreesClassifier(n_estimators=300, n_jobs=-1),\n",
        "], cover_X_train, cover_X_test, y_train.values, cv)"
      ],
      "execution_count":21,
      "outputs":[
        {
          "name":"stderr",
          "text":[
            "\r  0%|          | 0\/2 [00:00<?, ?it\/s]\r 50%|█████     | 1\/2 [00:01<00:01,  1.74s\/it]\r100%|██████████| 2\/2 [01:02<00:00, 36.61s\/it]\r100%|██████████| 2\/2 [01:02<00:00, 31.38s\/it]\n"
          ],
          "output_type":"stream"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "clf = LogisticRegression(penalty='none', solver='lbfgs')\n",
        "clf.fit(stacked_features_train, y_train)\n",
        "y_pred_clf = clf.predict(stacked_features_test)"
      ],
      "execution_count":22,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "f1_score(y_test, y_pred_clf)"
      ],
      "execution_count":23,
      "outputs":[
        {
          "data":{
            "text\/plain":[
              "0.7723502304147466"
            ]
          },
          "metadata":{
            
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "accuracy_score(y_test, y_pred_clf)"
      ],
      "execution_count":24,
      "outputs":[
        {
          "data":{
            "text\/plain":[
              "0.7267699115044248"
            ]
          },
          "metadata":{
            
          },
          "output_type":"display_data"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "## Пробуем XGboost classifier"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "import xgboost as xgb"
      ],
      "execution_count":25,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "### Преобразовываем независимые и целевую переменные"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "D_train = xgb.DMatrix(X_train, label=y_train)\n",
        "D_test = xgb.DMatrix(X_test, label=y_test)"
      ],
      "execution_count":26,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "### Настраиваем дефолтные параметры"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "param = {\n",
        "    'eta': 0.3, \n",
        "    'max_depth': 3,  \n",
        "    'objective': 'multi:softprob',  \n",
        "    'num_class': 3} \n",
        "\n",
        "steps = 20  # The number of training iterations"
      ],
      "execution_count":27,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "### Обучаем модель XGBoost Classifier"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "xgb_model = xgb.train(param, D_train, steps)"
      ],
      "execution_count":28,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "### Предсказываем целевую переменную"
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "preds = xgb_model.predict(D_test)\n",
        "best_preds = np.asarray([np.argmax(line) for line in preds])\n",
        "print(\"Accuracy = {}\".format(accuracy_score(y_test, best_preds)))"
      ],
      "execution_count":29,
      "outputs":[
        {
          "name":"stdout",
          "text":[
            "Accuracy = 0.7389380530973452\n"
          ],
          "output_type":"stream"
        }
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"code",
      "source":[
        "# Мы пытались\n",
        "# plt.figure(figsize=(10,7))\n",
        "# cmd_xgb = confusion_matrix(D_train, D_test)\n",
        "# sns.heatmap(cmd_xgb, annot=True, fmt='d')\n",
        "# plt.xlabel('Predicted')\n",
        "# plt.ylabel('Truth')\n",
        "# print('Accuracy XGBClassifier: ', cmd_xgb.score(D_train, y_test))"
      ],
      "execution_count":30,
      "outputs":[
        
      ],
      "metadata":{
        
      }
    },
    {
      "cell_type":"markdown",
      "source":[
        "# Всем спасибо, все идем спать!"
      ],
      "metadata":{
        
      }
    }
  ],
  "metadata":{
    
  },
  "nbformat":4,
  "nbformat_minor":0
}