#!/usr/bin/env python
# coding: utf-8

# # Linear regression :
# ### Regression or best fir, Least Sum of Squares Method
# equation of line, trasparent , y=mx+c or yhat=xb, y=yhat+e, Goodness of fit = rsqaure method or coefficient of determination

# In[61]:


get_ipython().run_line_magic('matplotlib', 'inline')
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
plt.rcParams['figure.figsize']=(20.0, 10.0)

#reading data
data = pd.read_csv("C:\Ashok\DS\Project_Data\headbrain.csv")


# In[62]:


data.shape, data.head(0)


# In[73]:


#collections x & y
X=data["Head Size(cm^3)"].values
Y=data["Brain Weight(grams)"].values
x=data["Head Size(cm^3)"].values
y=data["Brain Weight(grams)"].values


# In[74]:


#mean X and Y
mean_x=np.mean(X)
mean_y=np.mean(Y)

#total number of values
n=len(X)
n


# In[75]:


from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

# cannot use Rank 1 matrix in scikit learn
X=X.reshape((n,1))

#creating model
reg = LinearRegression()
#fitting training data
reg = reg.fit(X,Y)
# Y prediction
Y_predict=reg.predict(X)

#calculaton R2 score

r2_score = reg.score(X,Y)

print(r2_score)


# In[78]:


import statsmodels.api as sm


model = sm.OLS(x,y).fit()
predictions = model.predict(x)

print_model = model.summary()

print(model.summary())


# In[72]:


import pandas as pd


# In[39]:


data = {
    'month' : [1,2,3,4,5],
    'Free_time' : [120,110,100,90,80],
    'num_ve' : [8,8,7,4,5]
}

df = pd.DataFrame(data, columns=['month', 'Free_time', 'num_ve'])
df


# In[43]:


xx=df[['Free_time']]
xx


# In[45]:


xx=sm.add_constant(x)
xx


# # Multivariate regression
# ### data cleaning

# In[2]:


import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib
matplotlib.rcParams["figure.figsize"] = (20,10)


# In[3]:


df1=pd.read_csv("C:\Ashok\DS\Project_Data\Bengaluru_House_Data.csv")


# In[4]:


df1.head(0)


# In[5]:


df1.shape


# In[6]:


df1.groupby("area_type")["area_type"].agg('count')


# In[7]:


df2 = df1.drop(["area_type","availability","balcony","society"],axis="columns")


# In[8]:


df2.head()


# In[9]:


df2.isnull().sum()


# In[10]:


df3=df2.dropna()
df3.isnull().sum()


# In[11]:


df3["size"].unique()


# In[12]:


df3["bhk"]=df3['size'].apply(lambda x: int(x.split(" ")[0]))


# In[13]:


df3['bhk'].unique()


# In[14]:


df3[df3['bhk'] > 20]


# In[15]:


df3["total_sqft"].unique()


# In[16]:


# function to see the valuse the non float columns
def is_float(x):
    try:
        float(x)
    except:
       return False
    return True


# In[17]:


df3[~df3['total_sqft'].apply(is_float)].head()


# In[18]:


# 3 conditions are fullfilled, 1. convert '-' values to float, 2.retain float, 3. ignore rest of junk values
def convert_float(x):
    tokens=x.split('-')
    if len(tokens)==2:
        return (float(x.split('-')[0])+float(x.split('-')[1]))/2
    try:
        return float(x)
    except:
        return None


# In[19]:


df4=df3.copy()
df4['total_sqft']=df3['total_sqft'].apply(convert_float)


# In[20]:


df4.head()


# In[21]:


df4.drop('size',axis='columns').head()


# In[22]:


df4.loc[30]


# In[23]:


df4.iloc[4:5,0:2]


# ## Feature Engineering & dimensionality reduction techniques

# In[24]:


df5=df4.copy()


# In[25]:


df5.head(0)


# In[26]:


import math
df5['cost_per_sqf']=df5['price']*100000/df5['total_sqft']


# In[27]:


len(df5['location'].unique())


# In[28]:


df5.location=df5.location.apply(lambda x: x.strip())


# In[29]:


loca_stat = df5.groupby('location').location.agg('count').sort_values(ascending=False)


# In[30]:


len(loca_stat[loca_stat < 10 ])


# In[31]:


len(loca_stat[loca_stat < 10])


# In[32]:


loca_less = loca_stat[loca_stat < 10]


# In[33]:


len(df5.location.unique()), df5.shape


# In[34]:


df5.location=df5.location.apply(lambda x: 'other' if x in loca_less else x)


# In[35]:


df5.head()


# In[36]:


df5.head(0)


# #### Removal of outliers  , anamolies 

# In[39]:


df5[df5.total_sqft/df5.bhk < 300].head(2)


# In[45]:


df6=df5[~ (df5.total_sqft/df5.bhk < 300)]


# In[47]:


df6.shape


# In[48]:


df6.cost_per_sqf.describe()


# In[58]:


df6.groupby('location')['location','cost_per_sqf'].agg("mean").head()


# In[71]:


def remove_outliers(df):
    df_out = pd.DataFrame()
    for i, v in df.groupby('location'):
        m = np.mean(v.cost_per_sqf)
        st = np.std(v.cost_per_sqf)
        reduced_df = v[(v.cost_per_sqf > (m-st)) & (v.cost_per_sqf <= (m+st))]
        df_out = pd.concat([df_out,reduced_df],ignore_index=True)
    return df_out


# In[72]:


df7=remove_outliers(df6)


# In[73]:


df7.head()


# In[74]:


df7.shape


# In[76]:


def plot_scatter_chart(df,location):
    bhk2 = df[(df.location==location) & (df.bhk==2)]
    bhk3 = df[(df.location==location) & (df.bhk==3)]
    matplotlib.rcParams['figure.figsize'] = (15,10)
    plt.scatter(bhk2.total_sqft, bhk2.price,color='blue',label = '2 bhk', s=50)
    plt.scatter(bhk3.total_sqft, bhk3.price,marker='+',color='green',label = '3 bhk', s=50)
    plt.xlabel("Total Square Feet Area")
    plt.ylabel("Price")
    plt.title(location)
    plt.legend()
    

plot_scatter_chart(df7,"1st Block Jayanagar")


# In[82]:


df7.groupby('location')['location','price'].agg('count').sort_values('price',ascending=False).head()


# In[95]:


plot_scatter_chart(df7,"Hebbal")


# #### Remove the data points where price of 2bhk Greater then 3bhk

# In[89]:


def remove_bhk_outliers(df):
    exclude_indices = np.array([])
    for l, l_df in df.groupby('location'):
        bhk_stats = {}
        for i, v in l_df.groupby('bhk'):
            bhk_stats[i]={
                'mean' : np.mean(v.cost_per_sqf),
                 'std' : np.std(v.cost_per_sqf),
                 'count' : v.shape[0]
             }
        for s, t in l_df.groupby('bhk'):
            stats = bhk_stats.get(s-1)
            if stats and stats['count'] > 5:
                exclude_indices = np.append(exclude_indices, t[t.cost_per_sqf < (stats['mean'])].index.values)
    return df.drop(exclude_indices,axis='index')


# In[91]:


df8 = remove_bhk_outliers(df7)
df8.shape


# In[94]:


plot_scatter_chart(df8,"Hebbal")


# In[97]:


#Histogram

import matplotlib
matplotlib.rcParams["figure.figsize"] = (20,10)
plt.hist(df8.cost_per_sqf, rwidth=0.8)
plt.xlabel("Price per square feet")
plt.ylabel("count")


# In[96]:


df8.head(0)


# In[99]:


df8.bath.unique()


# In[100]:


df8[df8.bath > 10]


# In[102]:


plt.hist(df8.bath, rwidth=0.8)
plt.xlabel("No of bath")
plt.ylabel("Total count")


# In[104]:


df8[df8.bath > df8.bhk+2]


# In[106]:


df9 = df8[~ (df8.bath > df8.bhk+2)]


# In[108]:


df9.shape


# In[112]:


df10 = df9.drop(['size','cost_per_sqf'],axis='columns')


# In[113]:


df10.head()


# In[ ]:




