{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import itertools\n",
    "import datetime\n",
    "import exploration.dispersion as disp\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.formula.api as smf\n",
    "import analytics.tranche_data as tdata\n",
    "import ipysheet\n",
    "\n",
    "from analytics.basket_index import MarkitBasketIndex\n",
    "from analytics import on_the_run\n",
    "from statsmodels.graphics.regressionplots import plot_fit\n",
    "from scipy.special import logit, expit\n",
    "from pygam import LinearGAM, s, f, GAM\n",
    "from utils.db import dbengine, dbconn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "value_date = (datetime.datetime.today() - pd.offsets.BDay(1)).date()\n",
    "start = (datetime.datetime.today() - pd.offsets.BDay(1) * 365 *4).date()\n",
    "#end = (start + pd.offsets.BDay(1) * 365).date()\n",
    "end = datetime.datetime.today()\n",
    "gini_model, gini_results = {}, {}\n",
    "conn = dbconn(\"serenitasdb\")\n",
    "conn.autocommit = True\n",
    "for index_type in ['HY', 'IG', 'EU', 'XO']:\n",
    "    risk = disp.get_tranche_data(dbconn(\"serenitasdb\"), index_type)\n",
    "    risk = risk[risk.index_duration > 1] #filter out the short duration ones\n",
    "    gini_results[index_type], gini_model[index_type] = disp.create_models_v2(conn, risk)\n",
    "    #fitted = gini_model[index_type].fit()\n",
    "    #w = 1/(expit(fitted.fittedvalues + fitted.resid) -expit(fitted.fittedvalues))**2\n",
    "    #gini_results[index_type], gini_model[index_type] = disp.create_models_v2(conn, risk, w)\n",
    "gini_model['HY'].fit().summary()\n",
    "\n",
    "fieldlist = ['exp_percentage','dispersion','gini','tranche_loss_per','mispricing']\n",
    "for index_type in ['HY', 'IG', 'EU', 'XO']:\n",
    "    gini_results[index_type][fieldlist].to_csv('/home/serenitas/edwin/' + index_type + '_results.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fieldlist = ['exp_percentage','dispersion','gini','tranche_loss_per','mispricing']\n",
    "for index_type in ['HY', 'IG', 'EU', 'XO']:\n",
    "    gini_results[index_type][fieldlist].to_csv('/home/serenitas/edwin/' + index_type + '_results.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plot the residuals\n",
    "fitted = gini_model['HY'].fit()\n",
    "plt.figure(figsize=(8,5))\n",
    "p=plt.scatter(x=expit(fitted.fittedvalues),y=expit(fitted.fittedvalues + fitted.resid) -expit(fitted.fittedvalues),edgecolor='k')\n",
    "xmin=min(expit(fitted.fittedvalues))\n",
    "xmax = max(expit(fitted.fittedvalues))\n",
    "plt.hlines(y=0,xmin=xmin*0.9,xmax=xmax*1.1,color='red',linestyle='--',lw=3)\n",
    "plt.xlabel(\"Fitted values\",fontsize=15)\n",
    "plt.ylabel(\"Residuals\",fontsize=15)\n",
    "plt.title(\"Fitted vs. residuals plot\",fontsize=18)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plot the gini coefficients\n",
    "ginis = gini_results['HY'].xs([0, '5yr', 'HY'],level=['attach','tenor', 'index']).groupby(['date', 'series']).nth(-1).gini.unstack(level='series')\n",
    "ginis.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sheet = ipysheet.sheet(rows=2000, columns=7, column_headers=False, row_headers=False)\n",
    "import IPython\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "#IPython.OutputArea.auto_scroll_threshold = 20\n",
    "ginis.sort_index(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#look at Volatility vs Correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#use RFE to get the model instead\n",
    "from sklearn.preprocessing import PolynomialFeatures, PowerTransformer, normalize\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.feature_selection import RFECV, SelectKBest, f_regression\n",
    "from sklearn.compose import TransformedTargetRegressor\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "class MyTransformedTargetRegressor(TransformedTargetRegressor):\n",
    "    @property\n",
    "    def coef_(self):\n",
    "        return self.regressor_.coef_\n",
    "    \n",
    "    @property\n",
    "    def feature_importances_(self):\n",
    "        return self.regressor_.feature_importances_\n",
    "    \n",
    "index_type = 'HY'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_variable_list = ['tranche_loss_per', 'tranche_id', 'index_price', 'index_basis', 'detach', 'corr_at_detach', \n",
    "                     'attach_adj', 'detach_adj', 'index_theta', 'delta','gamma', 'corr01', 'expected_loss', \n",
    "                     'exp_percentage', 'indexfactor', 'duration', 'index_expected_loss',\n",
    "                     #Comment out to include\n",
    "    #                'index_duration',\n",
    "                     'theta',\n",
    "                     'cumulativeloss',\n",
    "    #                'att_moneyness',\n",
    "                    'det_moneyness',\n",
    "                     'thickness',\n",
    "    #                'moneyness',\n",
    "    #                'dispersion',\n",
    "                     'gini',\n",
    "                     'forward_delta']\n",
    "\n",
    "def run_rfe(index_type):\n",
    "    risk = disp.get_tranche_data(dbconn(\"serenitasdb\"), index_type)\n",
    "    attach_max = risk.index.get_level_values(\"attach\").max()\n",
    "    bottom_stack = risk[risk.index.get_level_values(\"attach\") != attach_max]\n",
    "    bottom_stack = bottom_stack[bottom_stack.tranche_loss_per > 0].dropna()\n",
    "\n",
    "    #prepare the variables\n",
    "    y = bottom_stack['tranche_loss_per']\n",
    "    X = bottom_stack.drop(all_variable_list, axis=1)\n",
    "\n",
    "    poly = PolynomialFeatures(3)\n",
    "    X = pd.DataFrame(PowerTransformer().fit_transform(X), index=X.index, columns=X.columns)\n",
    "    X_p = pd.DataFrame(poly.fit_transform(X), columns= poly.get_feature_names(X.columns))\n",
    "    regr = MyTransformedTargetRegressor(regressor=LinearRegression(), func=logit, inverse_func=expit)\n",
    "\n",
    "    rfecv = RFECV(regr).fit(X_p,y)\n",
    "\n",
    "    df = pd.merge(risk, \n",
    "                       pd.DataFrame(rfecv.predict(X_p), \n",
    "                                    index=X.index, \n",
    "                                    columns=['predict_tranche_loss']),\n",
    "                      how='left', left_index=True, right_index=True)\n",
    "\n",
    "    df.loc[df.index.get_level_values(\"attach\") != attach_max, \"predict_tranche_loss_per_index\"] = (\n",
    "        df.predict_tranche_loss * df.thickness / df.index_expected_loss\n",
    "    )\n",
    "\n",
    "    def aux(s):\n",
    "        temp = s.values\n",
    "        temp[-1] = 1 - temp[:-1].sum()\n",
    "        return temp\n",
    "\n",
    "    df[\"predict_tranche_loss_per_index\"] = df.groupby([\"index\", \"series\", \"date\"])[\"predict_tranche_loss_per_index\"].transform(aux)\n",
    "    df = df.assign(\n",
    "        mispricing=(df.exp_percentage - df.predict_tranche_loss_per_index)\n",
    "        * df.index_expected_loss\n",
    "        / (df.detach_adj - df.attach_adj)\n",
    "    )\n",
    "\n",
    "    print(index_type, \" Chosen columns: \", X_p[X_p.columns[rfecv.support_]].columns)\n",
    "    print(index_type, \" Score: \",  rfecv.score(X_p, y))\n",
    "    \n",
    "    return rfecv, df\n",
    "\n",
    "gini_model, gini_results = {}, {}\n",
    "for index_type in ['HY', 'IG', 'EU', 'XO']:\n",
    "    gini_model[index_type], gini_results[index_type] = run_rfe(index_type)\n",
    "fieldlist = ['exp_percentage','dispersion','gini','tranche_loss_per','mispricing']\n",
    "for index_type in ['HY', 'IG', 'EU', 'XO']:\n",
    "    gini_results[index_type][fieldlist].to_csv('/home/serenitas/edwin/' + index_type + '_results_rfecv.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gini_model, gini_results = {}, {}\n",
    "for index_type in ['HY', 'IG', 'EU', 'XO']:\n",
    "    gini_model[index_type], gini_results[index_type] = run_rfe(index_type)\n",
    "fieldlist = ['exp_percentage','dispersion','gini','tranche_loss_per','mispricing']\n",
    "for index_type in ['HY', 'IG', 'EU', 'XO']:\n",
    "    gini_results[index_type][fieldlist].to_csv('/home/serenitas/edwin/' + index_type + '_results_rfecv.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot number of features VS. cross-validation scores\n",
    "index_type = 'IG'\n",
    "plt.figure()\n",
    "plt.xlabel(\"Number of features selected\")\n",
    "plt.ylabel(\"Cross validation score (nb of correct classifications)\")\n",
    "plt.plot(range(1, len(gini_model[index_type].grid_scores_) + 1), gini_model[index_type].grid_scores_)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}