In [1]:
import pandas as pd
import numpy as np
from matplotlib.pyplot import subplots
In [17]:
x = np.linspace(0, 100, 300)
a = 0.7
b = 20
y = a*x + b
data = {'x': x, 'y': y}
df = pd.DataFrame(data)
df
Out[17]:
x y
0 0.000000 20.000000
1 0.334448 20.234114
2 0.668896 20.468227
3 1.003344 20.702341
4 1.337793 20.936455
... ... ...
295 98.662207 89.063545
296 98.996656 89.297659
297 99.331104 89.531773
298 99.665552 89.765886
299 100.000000 90.000000

300 rows × 2 columns

In [18]:
mean_x = df['x'].mean()
sd_x = df['x'].std()
mean_y = df['y'].mean()
sd_y = df['y'].std()

df1 = df.assign(x_in_std = (df['x'] - mean_x)/sd_x)
df2 = df1.assign(y_in_std = (df['y'] - mean_y)/sd_y)
df2
Out[18]:
x y x_in_std y_in_std
0 0.000000 20.000000 -1.723407 -1.723407
1 0.334448 20.234114 -1.711880 -1.711880
2 0.668896 20.468227 -1.700352 -1.700352
3 1.003344 20.702341 -1.688824 -1.688824
4 1.337793 20.936455 -1.677296 -1.677296
... ... ... ... ...
295 98.662207 89.063545 1.677296 1.677296
296 98.996656 89.297659 1.688824 1.688824
297 99.331104 89.531773 1.700352 1.700352
298 99.665552 89.765886 1.711880 1.711880
299 100.000000 90.000000 1.723407 1.723407

300 rows × 4 columns

In [10]:
print((df2['x_in_std']*df2['y_in_std']).sum()/df2.shape[0])

0.9966666666666667
In [19]:
df
Out[19]:
x y
0 0.000000 20.000000
1 0.334448 20.234114
2 0.668896 20.468227
3 1.003344 20.702341
4 1.337793 20.936455
... ... ...
295 98.662207 89.063545
296 98.996656 89.297659
297 99.331104 89.531773
298 99.665552 89.765886
299 100.000000 90.000000

300 rows × 2 columns

In [20]:
r = df['x'].corr(df['y'])
r
Out[20]:
np.float64(1.0)
In [21]:
df2
Out[21]:
x y x_in_std y_in_std
0 0.000000 20.000000 -1.723407 -1.723407
1 0.334448 20.234114 -1.711880 -1.711880
2 0.668896 20.468227 -1.700352 -1.700352
3 1.003344 20.702341 -1.688824 -1.688824
4 1.337793 20.936455 -1.677296 -1.677296
... ... ... ... ...
295 98.662207 89.063545 1.677296 1.677296
296 98.996656 89.297659 1.688824 1.688824
297 99.331104 89.531773 1.700352 1.700352
298 99.665552 89.765886 1.711880 1.711880
299 100.000000 90.000000 1.723407 1.723407

300 rows × 4 columns

In [22]:
fig, ax = subplots(figsize = (5, 5))
ax.scatter(df2['x_in_std'], df2['y_in_std'])
Out[22]:
<matplotlib.collections.PathCollection at 0x10c653190>
No description has been provided for this image
In [23]:
data = pd.read_csv('family_heights.csv')
data
Out[23]:
family father mother midparentHeight children childNum sex childHeight
0 1 78.5 67.0 75.43 4 1 male 73.2
1 1 78.5 67.0 75.43 4 2 female 69.2
2 1 78.5 67.0 75.43 4 3 female 69.0
3 1 78.5 67.0 75.43 4 4 female 69.0
4 2 75.5 66.5 73.66 4 1 male 73.5
... ... ... ... ... ... ... ... ...
929 203 62.0 66.0 66.64 3 1 male 64.0
930 203 62.0 66.0 66.64 3 2 female 62.0
931 203 62.0 66.0 66.64 3 3 female 61.0
932 204 62.5 63.0 65.27 2 1 male 66.5
933 204 62.5 63.0 65.27 2 2 female 57.0

934 rows × 8 columns

In [24]:
fig, ax = subplots(figsize = (5, 5))
ax.scatter(data['midparentHeight'], data['childHeight'])
Out[24]:
<matplotlib.collections.PathCollection at 0x10c6d9730>
No description has been provided for this image
In [25]:
r = data['midparentHeight'].corr(data['childHeight'])
r
Out[25]:
np.float64(0.32094989606395924)
In [34]:
mean_x = data['midparentHeight'].mean()
sd_x = data['midparentHeight'].std()
mean_y = data['childHeight'].mean()
sd_y = data['childHeight'].std()

data1 = data.assign(midp_in_std = (data['midparentHeight'] - mean_x)/sd_x)
data2 = data1.assign(child_in_std = (data['childHeight'] - mean_y)/sd_y)
data2
Out[34]:
family father mother midparentHeight children childNum sex childHeight midp_in_std child_in_std
0 1 78.5 67.0 75.43 4 1 male 73.2 3.452802 1.803190
1 1 78.5 67.0 75.43 4 2 female 69.2 3.452802 0.685637
2 1 78.5 67.0 75.43 4 3 female 69.0 3.452802 0.629760
3 1 78.5 67.0 75.43 4 4 female 69.0 3.452802 0.629760
4 2 75.5 66.5 73.66 4 1 male 73.5 2.470762 1.887006
... ... ... ... ... ... ... ... ... ... ...
929 203 62.0 66.0 66.64 3 1 male 64.0 -1.424110 -0.767180
930 203 62.0 66.0 66.64 3 2 female 62.0 -1.424110 -1.325957
931 203 62.0 66.0 66.64 3 3 female 61.0 -1.424110 -1.605345
932 204 62.5 63.0 65.27 2 1 male 66.5 -2.184220 -0.068710
933 204 62.5 63.0 65.27 2 2 female 57.0 -2.184220 -2.722897

934 rows × 10 columns

In [38]:
fig, ax = subplots(figsize = (5, 5))
ax.scatter(data2['midp_in_std'], data2['child_in_std'])
l = np.linspace(-4, 4, 100)
ax.plot(l, 1*l, color = 'green', linewidth = 3)
ax.plot(l, r*l, color = 'red', linewidth = 3)
Out[38]:
[<matplotlib.lines.Line2D at 0x10dd99400>]
No description has been provided for this image

Building a predictor¶

In [39]:
data
Out[39]:
family father mother midparentHeight children childNum sex childHeight
0 1 78.5 67.0 75.43 4 1 male 73.2
1 1 78.5 67.0 75.43 4 2 female 69.2
2 1 78.5 67.0 75.43 4 3 female 69.0
3 1 78.5 67.0 75.43 4 4 female 69.0
4 2 75.5 66.5 73.66 4 1 male 73.5
... ... ... ... ... ... ... ... ...
929 203 62.0 66.0 66.64 3 1 male 64.0
930 203 62.0 66.0 66.64 3 2 female 62.0
931 203 62.0 66.0 66.64 3 3 female 61.0
932 204 62.5 63.0 65.27 2 1 male 66.5
933 204 62.5 63.0 65.27 2 2 female 57.0

934 rows × 8 columns

In [40]:
def find_avg_ht(h):
    return(data[abs(data['midparentHeight'] - h) <= 0.5]['childHeight'].mean())

# Let's build our predictor
predicted_data = data.assign(predicted_height = data['midparentHeight'].apply(find_avg_ht))
predicted_data
Out[40]:
family father mother midparentHeight children childNum sex childHeight predicted_height
0 1 78.5 67.0 75.43 4 1 male 73.2 70.100000
1 1 78.5 67.0 75.43 4 2 female 69.2 70.100000
2 1 78.5 67.0 75.43 4 3 female 69.0 70.100000
3 1 78.5 67.0 75.43 4 4 female 69.0 70.100000
4 2 75.5 66.5 73.66 4 1 male 73.5 70.415789
... ... ... ... ... ... ... ... ... ...
929 203 62.0 66.0 66.64 3 1 male 64.0 65.156579
930 203 62.0 66.0 66.64 3 2 female 62.0 65.156579
931 203 62.0 66.0 66.64 3 3 female 61.0 65.156579
932 204 62.5 63.0 65.27 2 1 male 66.5 64.229630
933 204 62.5 63.0 65.27 2 2 female 57.0 64.229630

934 rows × 9 columns

In [41]:
fig, ax = subplots(figsize = (5, 5))
ax.scatter(predicted_data['midparentHeight'], predicted_data['childHeight'])
ax.scatter(predicted_data['midparentHeight'], predicted_data['predicted_height'], color = 'yellow')
Out[41]:
<matplotlib.collections.PathCollection at 0x10de31850>
No description has been provided for this image
In [43]:
slope = r*sd_y/sd_x
intercept = mean_y - slope*mean_x
Out[43]:
(np.float64(0.637360896969479), np.float64(22.63624054958975))
In [44]:
fig, ax = subplots(figsize = (5, 5))
ax.scatter(predicted_data['midparentHeight'], predicted_data['childHeight'])
ax.scatter(predicted_data['midparentHeight'], predicted_data['predicted_height'], color = 'yellow')
l = np.linspace(64, 80, 100)
ax.plot(l, slope*l+intercept, color='red', linewidth=3)
Out[44]:
[<matplotlib.lines.Line2D at 0x10dead5b0>]
No description has been provided for this image

Non Linear Regression¶

In [45]:
data = pd.read_csv('shotput.csv')
data
Out[45]:
Weight Lifted Shot Put Distance
0 37.5 6.4
1 51.5 10.2
2 61.3 12.4
3 61.3 13.0
4 63.6 13.2
5 66.1 13.0
6 70.0 12.7
7 92.7 13.9
8 90.5 15.5
9 90.5 15.8
10 94.8 15.8
11 97.0 16.8
12 97.0 17.1
13 97.0 17.8
14 102.0 14.8
15 102.0 15.5
16 103.6 16.1
17 100.4 16.2
18 108.4 17.9
19 114.0 15.9
20 115.3 15.8
21 114.9 16.7
22 114.7 17.6
23 123.6 16.8
24 125.8 17.0
25 119.1 18.2
26 118.9 19.2
27 141.1 18.6
In [47]:
fig, ax = subplots(figsize = (5, 5))
ax.scatter(data['Weight Lifted'], data['Shot Put Distance'])
Out[47]:
<matplotlib.collections.PathCollection at 0x10dff25b0>
No description has been provided for this image
In [49]:
r = data['Weight Lifted'].corr(data['Shot Put Distance'])
r
Out[49]:
np.float64(0.8929919094772171)
In [50]:
mean_x = data['Weight Lifted'].mean()
std_x = data['Weight Lifted'].std()

mean_y = data['Shot Put Distance'].mean()
std_y = data['Shot Put Distance'].std()

slope = r * (std_y/std_x)
intercept = mean_y - mean_x * slope

fig, ax = subplots(figsize = (5, 5))
ax.scatter(data['Weight Lifted'], data['Shot Put Distance'])
l = np.linspace(30, 150, 100)
ax.plot(l, slope*l+intercept, color = 'red', linewidth=3)
Out[50]:
[<matplotlib.lines.Line2D at 0x10e05a460>]
No description has been provided for this image
In [63]:
coefficients = np.polyfit(data['Weight Lifted'], data['Shot Put Distance'], 2)
coefficients
Out[63]:
array([-1.04004837e-03,  2.82708042e-01, -1.53182103e+00])
In [64]:
polynomial = np.poly1d(coefficients)
print(polynomial)

          2
-0.00104 x + 0.2827 x - 1.532
In [65]:
fig, ax = subplots(figsize = (5, 5))
ax.scatter(data['Weight Lifted'], data['Shot Put Distance'])
l = np.linspace(30, 150, 100)
y_poly = polynomial(l)
ax.plot(l, slope*l+intercept, color = 'red', linewidth=3)
ax.plot(l, y_poly, color = 'green', linewidth = 3)
Out[65]:
[<matplotlib.lines.Line2D at 0x10e2d0af0>]
No description has been provided for this image
In [ ]: