import pandas as pd
import numpy as np
from matplotlib.pyplot import subplots 

data = pd.read_csv('san_francisco_2019.csv')
data.shape

(44525, 8)

data['Total Compensation'].hist(bins = np.arange(0, 700000, 20000))

<Axes: >

data1 = data[data['Total Compensation'] >= 20000]
data1.shape

(36897, 8)

population_median = np.percentile(data1['Total Compensation'], 0.5)
population_median

np.float64(21125.0)

N = 500

sample_data = data1.sample(N, replace=False)
sample_data

num_resamples = 5000

L = []
for i in range(num_resamples):
    resample_data = sample_data.sample(N, replace=True)
    L.append(np.percentile(resample_data['Total Compensation'], 0.5))

lo = np.percentile(L, 2.5)
hi = np.percentile(L, 97.5)
lo, hi


fig, ax = subplots(figsize = (4, 4))
ax.hist(L);
ax.axvline(x=population_median, color='red')
ax.plot([lo, hi], [0, 0], color='yellow', linewidth=10)

[<matplotlib.lines.Line2D at 0x10b92dac0>]

num_experiments = 20
num_resamples = 3000
intervals = []

for i in range(num_experiments):
    sample_data = data1.sample(N, replace=False)
    L = []
    for i in range(num_resamples):
        resample_data = sample_data.sample(N, replace=True)
        L.append(np.percentile(resample_data['Total Compensation'], 0.5))
    lo = np.percentile(L, 2.5)
    hi = np.percentile(L, 97.5)
    intervals.append([lo, hi])

df = pd.DataFrame(intervals, columns=['lo', 'hi'])

fif, ax = subplots(figsize = (4, 4))
y = np.arange(1, len(df)+1)
ax.hlines(y, df['lo'], df['hi'], color='yellow', linewidth=3)
ax.axvline(population_median, color='red')

<matplotlib.lines.Line2D at 0x10ba1c610>

	Organization Group	Department	Job Family	Job	Salary	Overtime	Benefits	Total Compensation
30038	Community Health	Public Health	Housekeeping & Laundry	Porter	57119.0	13759.0	34116.0	104995.0
37651	Culture & Recreation	Public Library	Library	Librarian 1	98878.0	0.0	42716.0	141594.0
14934	Public Works, Transportation & Commerce	Municipal Transportation Agcy	Skilled Labor	Automotive Service Worker	74546.0	7822.0	36252.0	118619.0
16731	Public Works, Transportation & Commerce	Municipal Transportation Agcy	Street Transit	Transit Operator	84810.0	34789.0	41757.0	161356.0
15983	Public Works, Transportation & Commerce	Municipal Transportation Agcy	Street Transit	Transit Operator	81110.0	3888.0	38590.0	123588.0
...	...	...	...	...	...	...	...	...
10010	Public Works, Transportation & Commerce	Airport Commission	Housekeeping & Laundry	Custodian	71927.0	3313.0	34760.0	110000.0
33645	Community Health	Public Health	Nursing	Registered Nurse	188656.0	19169.0	58913.0	266739.0
18234	Public Works, Transportation & Commerce	Municipal Transportation Agcy	Semi-Skilled & General Labor	Track Maintenance Worker	81248.0	47970.0	41478.0	170697.0
10328	Public Works, Transportation & Commerce	Airport Commission	Airport Operation	Airport Security ID Technician	68402.0	442.0	33655.0	102499.0
16644	Public Works, Transportation & Commerce	Municipal Transportation Agcy	Street Transit	Transit Operator	73782.0	11242.0	37663.0	122686.0

Lecture 5¶