import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.pyplot import subplots

# The sklearn.datasets module in the scikit-learn library provides 
# utilities to load various datasets, which are useful for demonstrating 
# machine learning algorithms and benchmarking.
from sklearn.datasets import load_iris, load_wine, load_breast_cancer, load_digits

# The sklearn.model_selection module in the scikit-learn library provides 
# tools for model selection (choosing the best model) and hyperparameter 
# tuning, primarily through cross-validation and various search methods.
# We will use this for creating a test-train split and for cross-validation
from sklearn.model_selection import train_test_split, cross_val_score

# The sklearn.pipeline.Pipeline class in scikit-learn is a powerful 
# utility that sequentially chains multiple data transformation steps 
# and an optional final machine learning estimator into a single, unified object.
from sklearn.pipeline import Pipeline

# The sklearn.preprocessing package in scikit-learn provides a variety of 
# functions and transformer classes to prepare raw data for machine 
# learning models. These tools handle common tasks such as scaling, 
# transforming, and encoding features.
from sklearn.preprocessing import StandardScaler, MinMaxScaler


# The sklearn.neighbors module in the scikit-learn library provides 
# functionality for various neighbors-based machine learning methods.
from sklearn.neighbors import KNeighborsClassifier

# The sklearn.metrics module provides comprehensive tools for evaluating 
# machine learning models, including classification, regression, and 
# clustering performance. Key metrics include accuracy, precision, 
# recall, and F1-score for classification; MSE/MAE for regression; 
# and specialized tools like ROC AUC, confusion matrices, and pairwise 
# distance metrics.
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, ConfusionMatrixDisplay

# initialize the pseudo-random number generator (PRNG)
np.random.seed(42)

# We can load this dataset using load_iris
iris = load_iris(as_frame=True)
X_iris = iris.data #The feature matrix
y_iris = iris.target # The class label vector

# You can see the description of the data set using the command below
#print(iris.DESCR)

# Some of the properties of the dataset can be seen using the following print commands
print("Feature names:", list(X_iris.columns))
print("Target names:", list(iris.target_names))
print("Shape of X:", X_iris.shape)
print("Shape of y:", y_iris.shape)

Feature names: ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']
Target names: [np.str_('setosa'), np.str_('versicolor'), np.str_('virginica')]
Shape of X: (150, 4)
Shape of y: (150,)

# Create the train-test split. With test_size=0.30, it will create a 70:30 split, 
# with 70% of the data being used as training data.
X_train, X_test, y_train, y_test = train_test_split(
    X_iris, y_iris, test_size=0.30, random_state=42, stratify=y_iris
)

# We now set up the KNN model with k=3 and fit the test data.
# Note that in this implementation, we have not scaled the data.
# Sometimes, the KNN classifier behaves well even without scaling 
# (the iris dataset being one of the examples). Later, we'll see datasets
# on which the knn classifier does not behave well without scaling.
knn_iris = KNeighborsClassifier(n_neighbors=3)
knn_iris.fit(X_train, y_train)

# We use our trained KNN model to make the predictions on the test part of the 
# data that we held out.
y_pred = knn_iris.predict(X_test)
#print(y_pred)

# We can now calculate the fraction of correctly predicted labels.
acc = accuracy_score(y_test, y_pred)
print("Test accuracy:", acc)

Test accuracy: 0.9555555555555556

disp = ConfusionMatrixDisplay.from_estimator(knn_iris, X_test, y_test)
plt.show()

print(classification_report(y_test, y_pred, target_names=iris.target_names))

              precision    recall  f1-score   support

      setosa       1.00      1.00      1.00        15
  versicolor       0.88      1.00      0.94        15
   virginica       1.00      0.87      0.93        15

    accuracy                           0.96        45
   macro avg       0.96      0.96      0.96        45
weighted avg       0.96      0.96      0.96        45

# Use this code cell to write the code for the above task. You can create more code/markdown cells if required.

# Let us first load the wine dataset and observe the numerical values of the features
wine = load_wine(as_frame=True)
X_wine = wine.data
y_wine = wine.target

print("Shape of X:", X_wine.shape)
print("Feature names:", list(X_wine.columns))
print("Target names:", list(wine.target_names))
X_wine.describe().T[["mean", "std", "min", "max"]]

Shape of X: (178, 13)
Feature names: ['alcohol', 'malic_acid', 'ash', 'alcalinity_of_ash', 'magnesium', 'total_phenols', 'flavanoids', 'nonflavanoid_phenols', 'proanthocyanins', 'color_intensity', 'hue', 'od280/od315_of_diluted_wines', 'proline']
Target names: [np.str_('class_0'), np.str_('class_1'), np.str_('class_2')]

# We shall first create a train-test split
X_train, X_test, y_train, y_test = train_test_split(
    X_wine, y_wine, test_size=0.30, random_state=42, stratify=y_wine
)

# Let's find the accuracy of our 5-NN classifier without scaling
knn_raw = KNeighborsClassifier(n_neighbors=5)
knn_raw.fit(X_train, y_train)
pred_raw = knn_raw.predict(X_test)
acc_raw = accuracy_score(y_test, pred_raw)

# We shall use the StandardScaler to scale the training dataset
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
# The transform method is then used to scale the test data using parameters
# obtained from the training data.
X_test_scaled = scaler.transform(X_test)
# We now classify and evaluate using the scaled data
knn_scaled = KNeighborsClassifier(n_neighbors=5)
knn_scaled.fit(X_train_scaled, y_train)
pred_scaled = knn_scaled.predict(X_test_scaled)
acc_scaled = accuracy_score(y_test, pred_scaled)

print('Accuracy without scaling: ', acc_raw)
print('Accuracy with standard scaling: ', acc_scaled)

Accuracy without scaling:  0.7222222222222222
Accuracy with standard scaling:  0.9444444444444444

# We shall use the MinMaxScaler to scale the training dataset
scaler = MinMaxScaler()
X_train_mmscaled = scaler.fit_transform(X_train)
# The transform method is then used to scale the test data using parameters
# obtained from the training data.
X_test_mmscaled = scaler.transform(X_test)
# We now classify and evaluate using the scaled data
knn_mmscaled = KNeighborsClassifier(n_neighbors=5)
knn_mmscaled.fit(X_train_mmscaled, y_train)
pred_mmscaled = knn_mmscaled.predict(X_test_mmscaled)
acc_mmscaled = accuracy_score(y_test, pred_mmscaled)

print('Accuracy with min-max scaling: ', acc_mmscaled)

Accuracy with min-max scaling:  0.9629629629629629

# Let us also see the confusion matrix for the classifier using min-max scaling
disp = ConfusionMatrixDisplay.from_estimator(knn_mmscaled, X_test_mmscaled, y_test)
plt.show()

print(classification_report(y_test, pred_mmscaled, target_names=wine.target_names))

              precision    recall  f1-score   support

     class_0       1.00      1.00      1.00        18
     class_1       1.00      0.90      0.95        21
     class_2       0.88      1.00      0.94        15

    accuracy                           0.96        54
   macro avg       0.96      0.97      0.96        54
weighted avg       0.97      0.96      0.96        54

df = pd.read_csv('whitewinequality.csv')
df

cancer = load_breast_cancer(as_frame=True)
X_cancer = cancer.data
y_cancer = cancer.target

print("Shape of X:", X_cancer.shape)
print("Class names:", list(cancer.target_names))
print(y_cancer.value_counts())

Shape of X: (569, 30)
Class names: [np.str_('malignant'), np.str_('benign')]
target
1    357
0    212
Name: count, dtype: int64

# We shall first create a train-test split and use the training set for cross-validation
X_train, X_test, y_train, y_test = train_test_split(
    X_cancer, y_cancer, test_size=0.30, random_state=42, stratify=y_cancer
)

model = Pipeline([
        ("scaler", StandardScaler()),
        ("knn", KNeighborsClassifier(n_neighbors=5))
    ])
scores = cross_val_score(model, X_train, y_train, cv=5, scoring="accuracy")
print('The scores are:', scores)
print('The average score is: ', scores.mean())

The scores are: [0.9625     0.975      0.95       0.97468354 0.97468354]
The average score is:  0.967373417721519

# Use this code cell for the above task. Create more cells if needed.

X_train, X_test, y_train, y_test = train_test_split(
    X_wine, y_wine, test_size=0.30, random_state=44, stratify=y_wine
)
# Write the rest of the code for the above task here. Create more cells if required.

# Let us create a copy of the IRIS data that we'll be working with
X_base = X_iris.copy()
y_base = y_iris.copy()

# This is a helper function that adds normally distributed noise
# The number of normally distributed features is given as input
def add_noise_features(X, num_noise_features, random_state=42):
    rng = np.random.default_rng(random_state)
    noise = rng.normal(size=(X.shape[0], num_noise_features))
    noise_cols = [f"noise_{i+1}" for i in range(num_noise_features)]
    noise_df = pd.DataFrame(noise, columns=noise_cols, index=X.index)
    return pd.concat([X, noise_df], axis=1)

# Let us use this function to create a dataframe with 5 extra columns with noise
X_aug = add_noise_features(X_base, 5, random_state=42)
X_aug

# Write the code for the above task here. You may create more cells if needed.

from sklearn.datasets import fetch_covtype

X, y = fetch_covtype(return_X_y=True, shuffle=True, random_state=42)

# create a smaller subset of 20,000 examples
X_small, _, y_small, _ = train_test_split(
    X, y, train_size=20000, stratify=y, random_state=42
)
# For this project you are asked to consider (X_small, y_small) as the entire labeled dataset available to you.
# Please feel free to create as many cells as required for this project

	mean	std	min	max
alcohol	13.000618	0.811827	11.03	14.83
malic_acid	2.336348	1.117146	0.74	5.80
ash	2.366517	0.274344	1.36	3.23
alcalinity_of_ash	19.494944	3.339564	10.60	30.00
magnesium	99.741573	14.282484	70.00	162.00
total_phenols	2.295112	0.625851	0.98	3.88
flavanoids	2.029270	0.998859	0.34	5.08
nonflavanoid_phenols	0.361854	0.124453	0.13	0.66
proanthocyanins	1.590899	0.572359	0.41	3.58
color_intensity	5.058090	2.318286	1.28	13.00
hue	0.957449	0.228572	0.48	1.71
od280/od315_of_diluted_wines	2.611685	0.709990	1.27	4.00
proline	746.893258	314.907474	278.00	1680.00

	fixed acidity	volatile acidity	citric acid	residual sugar	chlorides	free sulfur dioxide	total sulfur dioxide	density	pH	sulphates	alcohol	quality
0	7.0	0.27	0.36	20.7	0.045	45.0	170.0	1.00100	3.00	0.45	8.8	6
1	6.3	0.30	0.34	1.6	0.049	14.0	132.0	0.99400	3.30	0.49	9.5	6
2	8.1	0.28	0.40	6.9	0.050	30.0	97.0	0.99510	3.26	0.44	10.1	6
3	7.2	0.23	0.32	8.5	0.058	47.0	186.0	0.99560	3.19	0.40	9.9	6
4	7.2	0.23	0.32	8.5	0.058	47.0	186.0	0.99560	3.19	0.40	9.9	6
...	...	...	...	...	...	...	...	...	...	...	...	...
4893	6.2	0.21	0.29	1.6	0.039	24.0	92.0	0.99114	3.27	0.50	11.2	6
4894	6.6	0.32	0.36	8.0	0.047	57.0	168.0	0.99490	3.15	0.46	9.6	5
4895	6.5	0.24	0.19	1.2	0.041	30.0	111.0	0.99254	2.99	0.46	9.4	6
4896	5.5	0.29	0.30	1.1	0.022	20.0	110.0	0.98869	3.34	0.38	12.8	7
4897	6.0	0.21	0.38	0.8	0.020	22.0	98.0	0.98941	3.26	0.32	11.8	6

	sepal length (cm)	sepal width (cm)	petal length (cm)	petal width (cm)	noise_1	noise_2	noise_3	noise_4	noise_5
0	5.1	3.5	1.4	0.2	0.304717	-1.039984	0.750451	0.940565	-1.951035
1	4.9	3.0	1.4	0.2	-1.302180	0.127840	-0.316243	-0.016801	-0.853044
2	4.7	3.2	1.3	0.2	0.879398	0.777792	0.066031	1.127241	0.467509
3	4.6	3.1	1.5	0.2	-0.859292	0.368751	-0.958883	0.878450	-0.049926
4	5.0	3.6	1.4	0.2	-0.184862	-0.680930	1.222541	-0.154529	-0.428328
...	...	...	...	...	...	...	...	...	...
145	6.7	3.0	5.2	2.3	1.461179	0.282587	0.767317	-1.140161	-1.119536
146	6.3	2.5	5.0	1.9	0.447814	0.058274	0.548739	-0.187671	0.278144
147	6.5	3.0	5.2	2.0	0.158119	0.777767	0.807008	-1.619872	-2.247269
148	6.2	3.4	5.4	2.3	1.001745	1.187725	-1.020623	-1.859835	0.099035
149	5.9	3.0	5.1	1.8	0.930838	1.797595	0.516298	-0.371717	-0.893131

Lab-8 (k-NN Classification)¶

(Intro. to Data Science: IIT Delhi - Abu Dhabi)¶

8.1 Introduction¶

Loading and understanding data¶

Training¶

Evaluation¶

Confusion Matrix, Precision, Recall, F1-Score, and Support¶

How to read the confusion matrix¶

Precision, Recall, F1-Score, and Support¶

1. Precision¶

2. Recall¶

3. F1-score¶

4. Support¶

Example calculations¶

Class: Cat¶

Summary table¶

Why these measures are useful¶

Task 8.1 (10 points):¶

8.2 Relevance of scaling¶

Task 8.2 (10 points):¶

8.3 Choosing $k$ using cross-validation¶

m-Fold Cross-Validation¶

Why it is useful¶

Example in k-NN¶

A note of caution¶

Choosing the best $k$ on Breast Cancer Dataset¶

Task 8.3 (10 points):¶

8.4 Weighted k-NN versus ordinary k-NN¶

Task 8.4 (10 points):¶

8.5 Curse of dimensionality: adding irrelevant features¶

Task 8.5 (10 points):¶

8.6 A minor project on k-NN classification¶

Task 8.6 (50 points):¶