Régression évoluée

# Advanced Regression
 
#Importing the required libraries
 
import warnings
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.feature_selection import RFE
from sklearn.metrics import r2_score
 
warnings.filterwarnings('ignore')
 
# Performing baisc checks in the dataset
 
house_price = pd.read_csv('train.csv')
 
print(house_price.shape)
 
print(house_price.info())
 
print(house_price.head())
 
print(house_price.describe([0.25,0.50,0.75,0.99]))
 
# Checking the dataset for the amount of nulls present
 
print(round(house_price.isnull().sum()/len(house_price.index),2).sort_values(ascending=False).head(18))
 
# Considering 10% as my threshold and dropping the column having more then the threshold
 
print(round(house_price.isnull().sum()/len(house_price.index),2)[round(house_price.isnull().sum()/
                                                                 len(house_price.index),2).values>0.10])
house_price = house_price.drop(['LotFrontage','Alley','FireplaceQu','PoolQC','Fence','MiscFeature','MoSold'],axis='columns')
 
# Checking the columns where the missing values between 0-10%
 
print(round(house_price.isnull().sum()/len(house_price.index),2)[round(house_price.isnull().sum()/
                                                                 len(house_price.index),2).values>0.00])
# Before going further we will try to convert the Year columns with the age where we are going to fill these columns with number , And if we query the max year for all these columns these will come out to be 2010 ,EX suppose the YearBuilt=2000 , Then YearBuiltOld = 2010-2000 =10
 
house_price['YearBuilt_Old'] = house_price.YearBuilt.max()-house_price.YearBuilt
house_price['YearRemodAdd_Old'] = house_price.YearRemodAdd.max()-house_price.YearRemodAdd
house_price['GarageYrBlt_Old'] = house_price.GarageYrBlt.max()-house_price.GarageYrBlt
house_price['YrSold_Old'] = house_price.YrSold.max()-house_price.YrSold
print(house_price[['YearBuilt','YearRemodAdd','GarageYrBlt','YrSold','YearBuilt_Old','YearRemodAdd_Old',
             'GarageYrBlt_Old','YrSold_Old']].sample(10))
 
# Lets drop the actual Year columns
 
house_price = house_price.drop(['YearBuilt','YearRemodAdd','GarageYrBlt','YrSold'],axis='columns')
 
# Imputing missing value
 
# Notice imputing GarageYrBlt_Old with -1 as these house donot have garage
 
house_price.MasVnrType.fillna('None',inplace=True)
house_price.MasVnrArea.fillna(house_price.MasVnrArea.mean(),inplace=True)
house_price.BsmtQual.fillna('TA',inplace=True)
house_price.BsmtCond.fillna('TA',inplace=True)
house_price.BsmtExposure.fillna('No',inplace=True)
house_price.BsmtFinType1.fillna('Unf',inplace=True)
house_price.BsmtFinType2.fillna('Unf',inplace=True)
house_price.GarageType.fillna('Attchd',inplace=True)
house_price.GarageYrBlt_Old.fillna(-1,inplace=True)
house_price.GarageFinish.fillna('Unf',inplace=True)
house_price.GarageQual.fillna('TA',inplace=True)
house_price.GarageCond.fillna('TA',inplace=True)
 
# As per the analysis 'Street' & 'Utilities' is explaining the lowest valrience hence dropping these columns
 
house_price.Street.value_counts()
house_price.Utilities.value_counts()
house_price = house_price.drop(['Street','Utilities'],axis='columns')
 
# 'Id' column is also of no use for our analysis hence dropping the column
 
house_price = house_price.drop('Id',axis='columns')
 
print(house_price[list(house_price.dtypes[house_price.dtypes!='object'].index)].describe())
 
# sns.boxplot(y = house_price['PoolArea'])
# house_price['WoodDeckSF'].value_counts()
 
# Lets plot some graph for the EDA purpose
 
plt.figure(figsize=(16,8))
plt.subplot(2,3,1)
plt.scatter(house_price.MasVnrArea,house_price.SalePrice)
plt.subplot(2,3,2)
plt.scatter(house_price.TotalBsmtSF,house_price.SalePrice)
plt.subplot(2,3,3)
plt.scatter(house_price['1stFlrSF'],house_price.SalePrice)
plt.subplot(2,3,4)
plt.scatter(house_price['GarageArea'],house_price.SalePrice)
plt.subplot(2,3,5)
plt.scatter(house_price['GrLivArea'],house_price.SalePrice)
plt.subplot(2,3,6)
plt.scatter(house_price['WoodDeckSF'],house_price.SalePrice)
 
# Plotting heatmap to check the corellation between varables
 
plt.figure(figsize=(16,16))
sns.heatmap(house_price[list(house_price.dtypes[house_price.dtypes!='object'].index)].corr(),annot=True)
plt.show()
 
# Below function is used for hadling the outliers where i am taking the lower and upper quantile as 0.25 & 0.99 respectively
 
print(house_price.shape)
 
num_col = list(house_price.dtypes[house_price.dtypes !='object'].index)
num_col = ['LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','TotalBsmtSF','1stFlrSF','GrLivArea','OpenPorchSF',
           'EnclosedPorch','3SsnPorch',
           'ScreenPorch' ,'PoolArea','MiscVal','SalePrice']
def drop_outliers(x):
    for col in num_col:
        Q1 = x[col].quantile(.25)
        Q3 = x[col].quantile(.99)
        IQR = Q3-Q1
        x =  x[(x[col] >= (Q1-(1.5*IQR))) & (x[col] <= (Q3+(1.5*IQR)))] 
    return x   
 
house_price = drop_outliers(house_price)
 
print(house_price.shape)
 
print(house_price[list(house_price.dtypes[house_price.dtypes=='object'].index)].head())
 
# Lets check for the below columns here we can clearly see that these are having some kind of order and hence we can say these are ordinal in nature
 
print(house_price[['LandSlope','ExterQual','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
            'HeatingQC','CentralAir',  'KitchenQual','GarageFinish','GarageQual','GarageCond',
             'ExterCond','LotShape']].head())
 
house_price['LandSlope'] = house_price.LandSlope.map({'Gtl':0,'Mod':1,'Sev':2})
house_price['ExterQual'] = house_price.ExterQual.map({'Po':0,'Fa':1,'TA':2,'Gd':3,'Ex':4})
house_price['BsmtQual'] = house_price.BsmtQual.map({'NA':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
house_price['BsmtCond'] = house_price.BsmtCond.map({'NA':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
house_price['BsmtExposure'] = house_price.BsmtExposure.map({'NA':0,'No':1,'Mn':2,'Av':3,'Gd':4})
house_price['BsmtFinType1'] = house_price.BsmtFinType1.map({'NA':0,'Unf':1,'LwQ':2,'Rec':3,'BLQ':4,'ALQ':5,'GLQ':6})
house_price['BsmtFinType2'] = house_price.BsmtFinType2.map({'NA':0,'Unf':1,'LwQ':2,'Rec':3,'BLQ':4,'ALQ':5,'GLQ':6})
house_price['HeatingQC'] = house_price.HeatingQC.map({'Po':0,'Fa':1,'TA':2,'Gd':3,'Ex':4})
house_price['CentralAir'] = house_price.CentralAir.map({'N':0,'Y':1})
house_price['KitchenQual'] = house_price.KitchenQual.map({'Po':0,'Fa':1,'TA':2,'Gd':3,'Ex':4})
house_price['GarageFinish'] = house_price.GarageFinish.map({'NA':0,'Unf':1,'RFn':2,'Fin':3})
house_price['GarageQual'] = house_price.GarageQual.map({'NA':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
house_price['GarageCond'] = house_price.GarageCond.map({'NA':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
house_price['ExterCond'] = house_price.ExterCond.map({'Po':0,'Fa':1,'TA':2,'Gd':3,'Ex':4})
house_price['LotShape'] = house_price.LotShape.map({'IR1':0,'IR2':1,'IR3':2,'Reg':3})
 
# Lets take a look into the converted columns
 
print(house_price[['LandSlope','ExterQual','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
            'HeatingQC','CentralAir',  'KitchenQual','GarageFinish','GarageQual','GarageCond',
             'ExterCond','LotShape']].head())
 
# Creating and joining dummy column with the actual dataset
 
dummy_col = pd.get_dummies(house_price[['MSZoning','LandContour','LotConfig','Neighborhood','Condition1','Condition2','BldgType',
             'HouseStyle','RoofStyle','RoofMatl','Exterior1st',  'Exterior2nd','MasVnrType','Foundation',
             'Heating','Electrical','Functional','GarageType','PavedDrive','SaleType','SaleCondition']],
                           drop_first=True)
 
house_price = pd.concat([house_price,dummy_col],axis='columns')
 
house_price = house_price.drop(['MSZoning','LandContour','LotConfig','Neighborhood','Condition1','Condition2','BldgType',
             'HouseStyle','RoofStyle','RoofMatl','Exterior1st',  'Exterior2nd','MasVnrType','Foundation',
             'Heating','Electrical','Functional','GarageType','PavedDrive','SaleType','SaleCondition'],axis='columns')
 
# Let us check the distribution of our target variable before scaling and Splitting
 
plt.figure(figsize=(16,6))
sns.distplot(house_price.SalePrice)
plt.show()
 
# Creating train and test dataset for validation purpose
 
df_train,df_test = train_test_split(house_price,train_size=0.7,test_size=0.3,random_state=42)
 
print(house_price[['LandSlope','ExterQual','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
            'HeatingQC','CentralAir',  'KitchenQual','GarageFinish','GarageQual','GarageCond',
             'ExterCond','LotShape']].head())
 
# Scaling the train dataset
 
# Note as scale of our dependent valriable SalePrice is very different with the independent variable i am scaling the dependent variable
 
num_col = ['MSSubClass','LotArea','OverallQual','OverallCond',
           'MasVnrArea','BsmtFinSF1',
           'BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF','2ndFlrSF',
           'LowQualFinSF','GrLivArea','BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr',
           'KitchenAbvGr','TotRmsAbvGrd','Fireplaces','GarageCars',
           'GarageArea','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch',
           'ScreenPorch','PoolArea','MiscVal','SalePrice']
 
# Lets check the distribution again after scaling
 
scaler = StandardScaler()
df_train[num_col] = scaler.fit_transform(df_train[num_col])
df_test[num_col] = scaler.transform(df_test[num_col])
 
plt.figure(figsize=(16,6))
plt.subplot(121)
sns.distplot(df_train.SalePrice)
plt.subplot(122)
sns.distplot(df_test.SalePrice)
 
# Spliting the dependent and independent variable
 
y_train = df_train.pop('SalePrice')
X_train = df_train
 
y_test = df_test.pop('SalePrice')
X_test = df_test
 
# Now using RFE lets try to to find the optimal number of feature
 
# Note : I cannot use RFE with GridSearchCV as 192 variable with 5 folds will create 960 fit and it will take more then 1 Hour to get the result so i am using RFE directly
 
print(len(X_train.columns))
 
lm  = LinearRegression()
lm.fit(X_train,y_train)
rfe = RFE(lm)
rfe.fit(X_train,y_train)
 
rfe_scores = pd.DataFrame(list(zip(X_train.columns,rfe.support_,rfe.ranking_)))
rfe_scores.columns = ['Column_Names','Status','Rank']
 
rfe_sel_columns = list(rfe_scores[rfe_scores.Status==True].Column_Names)
 
# Lets filter the train and test set for the RFE selected columns
 
X_train = X_train[rfe_sel_columns]
X_test = X_test[rfe_sel_columns]
 
# Lets try first with the Lasso regression model
 
lm = Lasso(alpha=0.001)
lm.fit(X_train,y_train)
 
y_train_pred = lm.predict(X_train)
print(r2_score(y_true=y_train,y_pred=y_train_pred))
 
y_test_pred  = lm.predict(X_test)
print(r2_score(y_true=y_test,y_pred=y_test_pred))
 
model_parameter = list(lm.coef_)
model_parameter.insert(0,lm.intercept_)
model_parameter = [round(x,3) for x in model_parameter]
col = df_train.columns
col.insert(0,'Constant')
print(list(zip(col,model_parameter)))
 
model_parameter = list(lm.coef_)
model_parameter.insert(0,lm.intercept_)
model_parameter = [round(x,3) for x in model_parameter]
col = df_train.columns
col.insert(0,'Constant')
list(zip(col,model_parameter))
 
# Now lets try to improve our model with the optimal value of alpha using GridSearchCV
 
folds = KFold(n_splits=10,shuffle=True,random_state=42)
 
hyper_param = {'alpha':[0.001, 0.01, 0.1,1.0, 5.0, 10.0,20.0]}
 
model = Lasso()
 
model_cv = GridSearchCV(estimator = model,
                        param_grid=hyper_param,
                        scoring='r2',
                        cv=folds,
                        verbose=1,
                        return_train_score=True
                       )
 
model_cv.fit(X_train,y_train)
 
cv_result_l = pd.DataFrame(model_cv.cv_results_)
cv_result_l['param_alpha'] = cv_result_l['param_alpha'].astype('float32')
print(cv_result_l.head())
 
plt.figure(figsize=(16,8))
plt.plot(cv_result_l['param_alpha'],cv_result_l['mean_train_score'])
plt.plot(cv_result_l['param_alpha'],cv_result_l['mean_test_score'])
plt.xscale('log')
plt.ylabel('R2 Score')
plt.xlabel('Alpha')
plt.show()
 
# Checking the best parameter(Alpha value)
print(model_cv.best_params_)
 
lasso = Lasso(alpha=0.001)
lasso.fit(X_train,y_train)
 
y_train_pred = lasso.predict(X_train)
y_test_pred = lasso.predict(X_test)
 
print(r2_score(y_true=y_train,y_pred=y_train_pred))
print(r2_score(y_true=y_test,y_pred=y_test_pred))
 
model_param = list(lasso.coef_)
model_param.insert(0,lasso.intercept_)
cols = df_train.columns
cols.insert(0,'const')
lasso_coef = pd.DataFrame(list(zip(cols,model_param)))
lasso_coef.columns = ['Featuere','Coef']
 
print(lasso_coef.sort_values(by='Coef',ascending=False).head(10))
 
# Now lets use the ridge regression
 
ridge = Ridge(alpha=0.001)
ridge.fit(X_train,y_train)
 
y_train_pred = ridge.predict(X_train)
print(r2_score(y_train,y_train_pred))
y_test_pred = ridge.predict(X_test)
print(r2_score(y_test,y_test_pred))
 
# As we can see the above alpha value is not optimal for ridge there are sign of overfitting the clear difference we can see in the train and test score
 
# Now lets try to improve our model with the optimal value of alpha using GridSearchCV
 
folds  = KFold(n_splits=10,shuffle=True,random_state=42)
 
hyper_param = {'alpha':[0.001,0.01,0.1,0.2,0.5,0.9,1.0, 5.0, 10.0,20.0]}
 
model = Ridge()
 
model_cv = GridSearchCV(estimator=model,
                        param_grid=hyper_param,
                        scoring='r2',
                        cv=folds,
                        verbose=1,
                        return_train_score=True)
 
model_cv.fit(X_train,y_train)
 
cv_result_r = pd.DataFrame(model_cv.cv_results_)
cv_result_r['param_alpha'] = cv_result_r['param_alpha'].astype('float32')
print(cv_result_r.head())
 
plt.figure(figsize=(16,8))
plt.plot(cv_result_r['param_alpha'],cv_result_r['mean_train_score'])
plt.plot(cv_result_r['param_alpha'],cv_result_r['mean_test_score'])
plt.xlabel('Alpha')
# plt.xscale('log')
plt.ylabel('R2 Score')
plt.show()
 
# On the basis of above graph lets create the model
 
# Checking the best parameter(Alpha value)
print(model_cv.best_params_)
 
ridge = Ridge(alpha = 0.9)
ridge.fit(X_train,y_train)
 
y_pred_train = ridge.predict(X_train)
print(r2_score(y_train,y_pred_train))
 
y_pred_test = ridge.predict(X_test)
print(r2_score(y_test,y_pred_test))
 
model_parameter = list(ridge.coef_)
model_parameter.insert(0,ridge.intercept_)
cols = df_train.columns
cols.insert(0,'constant')
ridge_coef = pd.DataFrame(list(zip(cols,model_parameter)))
ridge_coef.columns = ['Feaure','Coef']
 
print(ridge_coef.sort_values(by='Coef',ascending=False).head(10))
 
# After creating model in both Ridge and Lasso we can see that the r2_scores are almost same for both of them but as lasso will penalize more on the dataset and can also help in feature elemination i am goint to consider that as my final model.
 
# Final Model
 
lasso = Lasso(alpha=0.001)
lasso.fit(X_train,y_train)
 
y_train_pred = lasso.predict(X_train)
y_test_pred = lasso.predict(X_test)
 
print(r2_score(y_true=y_train,y_pred=y_train_pred))
print(r2_score(y_true=y_test,y_pred=y_test_pred))
 
# After compairing both the model we can see that the below Features are best explaining the DataSet
 
#MiscVal      : $Value of miscellaneous feature 
#BsmtHalfBath : Basement half bathrooms
#LowQualFinSF : Low quality finished square feet (all floors)
#BsmtFullBath : Basement full bathrooms
#HalfBath     : Half baths above grade
 
# Best alpha value for Lasso : {'alpha': 0.001}
# Best alpha value for Ridge : {'alpha': 0.9}