In [1]:
import pandas as pd
import numpy as np
from matplotlib.pyplot import subplots

house = pd.read_csv('house.csv')
house.head(3)
Out[1]:
Order PID MS SubClass MS Zoning Lot Frontage Lot Area Street Alley Lot Shape Land Contour ... Pool Area Pool QC Fence Misc Feature Misc Val Mo Sold Yr Sold Sale Type Sale Condition SalePrice
0 1 526301100 20 RL 141.0 31770 Pave NaN IR1 Lvl ... 0 NaN NaN NaN 0 5 2010 WD Normal 215000
1 2 526350040 20 RH 80.0 11622 Pave NaN Reg Lvl ... 0 NaN MnPrv NaN 0 6 2010 WD Normal 105000
2 3 526351010 20 RL 81.0 14267 Pave NaN IR1 Lvl ... 0 NaN NaN Gar2 12500 6 2010 WD Normal 172000

3 rows × 82 columns

In [2]:
house = house[house['Bldg Type'] == '1Fam']
house = house[house['Sale Condition'] == 'Normal']
sales = house[['SalePrice', '1st Flr SF', '2nd Flr SF', 'Total Bsmt SF', 
               'Garage Area', 'Wood Deck SF', 'Open Porch SF', 'Lot Area', 
               'Year Built', 'Yr Sold']]
sales.head(3)
Out[2]:
SalePrice 1st Flr SF 2nd Flr SF Total Bsmt SF Garage Area Wood Deck SF Open Porch SF Lot Area Year Built Yr Sold
0 215000 1656 0 1080.0 528.0 210 62 31770 1960 2010
1 105000 896 0 882.0 730.0 140 0 11622 1961 2010
2 172000 1329 0 1329.0 312.0 393 36 14267 1958 2010
In [3]:
n = sales.shape[0] # Number of observations
X = np.ones((n, 1))
X = np.insert(X, 1, sales['1st Flr SF'], axis=1)
X = np.insert(X, 2, sales['2nd Flr SF'], axis=1)
y = sales['SalePrice'].reset_index(drop=True)
X
Out[3]:
array([[1.000e+00, 1.656e+03, 0.000e+00],
       [1.000e+00, 8.960e+02, 0.000e+00],
       [1.000e+00, 1.329e+03, 0.000e+00],
       ...,
       [1.000e+00, 9.700e+02, 0.000e+00],
       [1.000e+00, 1.389e+03, 0.000e+00],
       [1.000e+00, 9.960e+02, 1.004e+03]])
In [4]:
Z = np.matmul(X.T, X)
t = np.linalg.inv(Z)@X.T@y
MSE = (1/n) * np.linalg.norm(y - X@t)**2
print('MSE = ', MSE)
print('Theta = ', t)

MSE =  1705349005.3197255
Theta =  [-29985.3612312     155.19406373     91.35680257]
In [7]:
learning_rate = 0.0000000001
t = np.zeros(3).T
for i in range(3):
    t = t + (2*learning_rate/n) * (X.T @ (y - X@t))
    loss = (1/n) * np.linalg.norm(y - X@t)**2

MSE = (1/n) * np.linalg.norm(y - X@t)**2
print('MSE, Theta:', MSE, t)

X

MSE, Theta: 37417784909.067505 [1.07477231e-04 1.33284713e-01 4.37624711e-02]
Out[7]:
array([[1.000e+00, 1.656e+03, 0.000e+00],
       [1.000e+00, 8.960e+02, 0.000e+00],
       [1.000e+00, 1.329e+03, 0.000e+00],
       ...,
       [1.000e+00, 9.700e+02, 0.000e+00],
       [1.000e+00, 1.389e+03, 0.000e+00],
       [1.000e+00, 9.960e+02, 1.004e+03]])
In [8]:
from sklearn.preprocessing import StandardScaler

# Scale the features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
X_scaled[:, 0] = np.ones(sales.shape[0])

Z = np.matmul(X_scaled.T, X_scaled)
t = np.linalg.inv(Z)@X_scaled.T@y
MSE = (1/n) * np.linalg.norm(y - X_scaled@t)**2
print('MSE = ', MSE)
print('Theta = ', t)

MSE =  1705349005.3197255
Theta =  [179184.62687313  56135.61010536  39157.22624338]
In [11]:
learning_rate = 0.01
t = np.zeros(3).T
for i in range(2000):
    t = t + (2*learning_rate/n) * (X_scaled.T @ (y - X_scaled@t))
    #MSE = (1/n) * np.linalg.norm(y - X_scaled@t)**2
    #print('RMSE, Theta:', MSE, t)

MSE = (1/n) * np.linalg.norm(y - X_scaled@t)**2
print('RMSE, Theta:', MSE, t)

RMSE, Theta: 1705349005.3197255 [179184.62687313  56135.61010536  39157.22624338]
In [17]:
# This is stochastic gradient descent
learning_rate = 0.01
t = np.zeros(3)
n = sales.shape[0]
for i in range(1200):
    for j in range(n):
        t = t + (2*learning_rate) * (y[j] - X_scaled[j, :]@t) * X_scaled[j, :]
        
MSE = (1/n) * np.linalg.norm(y - X_scaled@t)**2
print('RMSE, Theta:', MSE, t)

RMSE, Theta: 1766798902.7592735 [174477.21671957  49694.33814832  37747.92307174]
In [30]:
# This is batch gradient descent
learning_rate = 0.01
t = np.zeros(3)
n = sales.shape[0]
m=100
for i in range(100):
    for j in range(n//m):
        t = t + (2*learning_rate/m) * X_scaled[j*m:(j+1)*m, :].T @ (y[j*m:(j+1)*m] - X_scaled[j*m:(j+1)*m, :]@t)
        
MSE = (1/n) * np.linalg.norm(y - X_scaled@t)**2
print('RMSE, Theta:', MSE, t)

RMSE, Theta: 1705564912.1929908 [178949.12523594  55787.08837227  39290.02664459]
In [21]:
sales.corr()
Out[21]:
SalePrice 1st Flr SF 2nd Flr SF Total Bsmt SF Garage Area Wood Deck SF Open Porch SF Lot Area Year Built Yr Sold
SalePrice 1.000000 0.642466 0.357522 0.652979 0.638594 0.352699 0.336909 0.290823 0.565165 0.025949
1st Flr SF 0.642466 1.000000 -0.230561 0.765052 0.464566 0.247685 0.220888 0.303834 0.316339 0.026647
2nd Flr SF 0.357522 -0.230561 1.000000 -0.182213 0.172852 0.097926 0.227684 0.025948 0.079553 -0.020525
Total Bsmt SF 0.652979 0.765052 -0.182213 1.000000 0.456592 0.237535 0.204110 0.208933 0.425617 0.047675
Garage Area 0.638594 0.464566 0.172852 0.456592 1.000000 0.237945 0.240943 0.183717 0.487135 0.029293
Wood Deck SF 0.352699 0.247685 0.097926 0.237535 0.237945 1.000000 0.055236 0.127837 0.254986 0.019196
Open Porch SF 0.336909 0.220888 0.227684 0.204110 0.240943 0.055236 1.000000 0.095189 0.190891 -0.031687
Lot Area 0.290823 0.303834 0.025948 0.208933 0.183717 0.127837 0.095189 1.000000 0.083281 -0.008188
Year Built 0.565165 0.316339 0.079553 0.425617 0.487135 0.254986 0.190891 0.083281 1.000000 0.046000
Yr Sold 0.025949 0.026647 -0.020525 0.047675 0.029293 0.019196 -0.031687 -0.008188 0.046000 1.000000
In [29]:
from sklearn.linear_model import LinearRegression

model = LinearRegression()
model.fit(sales[['1st Flr SF']], sales['SalePrice'])
r_sq = model.score(sales[['1st Flr SF']], sales['SalePrice'])
print('One feature:', r_sq)

model = LinearRegression()
model.fit(sales[['1st Flr SF', '2nd Flr SF']], sales['SalePrice'])
r_sq = model.score(sales[['1st Flr SF', '2nd Flr SF']], sales['SalePrice'])
print('Two features:', r_sq)

model = LinearRegression()
model.fit(sales[['1st Flr SF', '2nd Flr SF', 'Year Built']], sales['SalePrice'])
r_sq = model.score(sales[['1st Flr SF', '2nd Flr SF', 'Year Built']], sales['SalePrice'])
print('Three features:', r_sq)

model = LinearRegression()
model.fit(sales[['1st Flr SF', '2nd Flr SF', 'Year Built', 'Garage Area']], sales['SalePrice'])
r_sq = model.score(sales[['1st Flr SF', '2nd Flr SF', 'Year Built', 'Garage Area']], sales['SalePrice'])
print('Four features:', r_sq)

One feature: 0.41276288766116953
Two features: 0.6827991925732996
Three features: 0.7726766859516288
Four features: 0.7880072396536026
In [ ]: