task2b.py

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import csv

from sklearn import datasets, cluster
from sklearn import neighbors
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn import preprocessing
from sklearn.preprocessing import PolynomialFeatures
from sklearn.tree import DecisionTreeClassifier
from sklearn.feature_selection import chi2
from sklearn.feature_selection import SelectKBest
from sklearn.decomposition import PCA
from sklearn.metrics.cluster import normalized_mutual_info_score
from sklearn.preprocessing import MinMaxScaler
from sklearn.cluster import KMeans

import scipy.stats as stats
from scipy.stats import chi2_contingency

##load in the data
life=pd.read_csv('life.csv',encoding = 'ISO-8859-1',na_values='..')
world=pd.read_csv('world.csv',encoding = 'ISO-8859-1',na_values='..')

result = pd.merge(life, world, on=['Country Code'])
result = result.sort_values(by='Country Code', ascending = True)

data=result[['Access to electricity (% of population) [EG.ELC.ACCS.ZS]',
 'Adjusted net national income per capita (current US$) [NY.ADJ.NNTY.PC.CD]',
 'Age dependency ratio (% of working-age population) [SP.POP.DPND]',
 'Cause of death, by communicable diseases and maternal, prenatal and nutrition conditions (% of total) [SH.DTH.COMM.ZS]',
 'Current health expenditure per capita (current US$) [SH.XPD.CHEX.PC.CD]',
 'Fertility rate, total (births per woman) [SP.DYN.TFRT.IN]',
 'Fixed broadband subscriptions (per 100 people) [IT.NET.BBND.P2]',
 'Fixed telephone subscriptions (per 100 people) [IT.MLT.MAIN.P2]',
 'GDP per capita (constant 2010 US$) [NY.GDP.PCAP.KD]',
 'GNI per capita, Atlas method (current US$) [NY.GNP.PCAP.CD]',
 'Individuals using the Internet (% of population) [IT.NET.USER.ZS]',
 'Lifetime risk of maternal death (%) [SH.MMR.RISK.ZS]',
 'People using at least basic drinking water services (% of population) [SH.H2O.BASW.ZS]',
 'People using at least basic drinking water services, rural (% of rural population) [SH.H2O.BASW.RU.ZS]',
 'People using at least basic drinking water services, urban (% of urban population) [SH.H2O.BASW.UR.ZS]',
 'People using at least basic sanitation services, urban (% of urban population) [SH.STA.BASS.UR.ZS]',
 'Prevalence of anemia among children (% of children under 5) [SH.ANM.CHLD.ZS]',
 'Secure Internet servers (per 1 million people) [IT.NET.SECR.P6]',
 'Self-employed, female (% of female employment) (modeled ILO estimate) [SL.EMP.SELF.FE.ZS]',
 'Wage and salaried workers, female (% of female employment) (modeled ILO estimate) [SL.EMP.WORK.FE.ZS]']].astype(float)

##get just the class labels
classlabel=result['Life expectancy at birth (years)']

X_train, X_test, y_train, y_test = train_test_split(data, classlabel, train_size=0.7, test_size=0.3, random_state=200)
X_train = X_train.fillna(X_train.median())
X_test = X_test.fillna(X_train.median())
med = X_train.median()

#normalise the data to have 0 mean and unit variance using the library functions.  This will help for later
#computation of distances between instances
scaler = preprocessing.StandardScaler().fit(X_train)
X_train=scaler.transform(X_train)
X_test=scaler.transform(X_test)

Sum_of_squared_distances = []
K = range(1,15)
for k in K:
    km = KMeans(n_clusters=k)
    km = km.fit(X_train)
    Sum_of_squared_distances.append(km.inertia_)
    
plt.plot(K, Sum_of_squared_distances, 'bx-')
plt.xlabel('k')
plt.ylabel('Sum_of_squared_distances')
plt.title('Elbow Method For Optimal k')
plt.savefig("task2bgraph1.png")
plt.show()

km = KMeans(n_clusters = 3).fit(X_train)
f_cluster_train = km.predict(X_train)
f_cluster_test = km.predict(X_test)


poly = PolynomialFeatures(2, include_bias=False, interaction_only=True)
poly.fit(X_train)
X_train_1=poly.transform(X_train)
X_test_1=poly.transform(X_test)

print("First five rows of the 190 features generated using interaction term pairs")
print(pd.DataFrame(X_train_1).iloc[:,20:].head(5))
print("\n")
print("First five rows of the 1 feature generated by clustering")
print(pd.DataFrame(f_cluster_train).head(5))
 

X_train_1 = np.concatenate((X_train_1,f_cluster_train[:,None]), axis=1)
X_test_1 = np.concatenate((X_test_1,f_cluster_test[:,None]), axis=1)

print("\n")
print("First five rows of the 211 features from interaction pairs and clustering before feature selection")
print(pd.DataFrame(X_train_1).head(5))

dep_features = []
for feature in range(0,211):
    x_val = X_train_1[:,feature]
    nmi = normalized_mutual_info_score(x_val, y_train)
    dep_features.append([feature, nmi]) 
dep_features = sorted(dep_features, key=lambda x: x[1], reverse = True)

top_features = []
print("\n")
print("Top 4 features with highest NMI: ")
for feature in dep_features[:4]:
    print("feature:",feature[0])
    print("nmi:",feature[1])
    top_features.append(feature[0])
X_train_1 = X_train_1[:,top_features]
X_test_1 = X_test_1[:,top_features]

print("\n")
print("First five rows of the 4 feature selected from 211 features")
print(pd.DataFrame(X_train_1).head(5))

knn = neighbors.KNeighborsClassifier(n_neighbors=3)
knn.fit(X_train_1, y_train)
y_pred_1=knn.predict(X_test_1)

pca = PCA(n_components=4)
pca.fit(X_train)
X_train_2=pca.transform(X_train)
X_test_2=pca.transform(X_test)

print("\n")
print("First five rows of the 4 features generated by PCA")
print(pd.DataFrame(X_train_2).head(5))

knn = neighbors.KNeighborsClassifier(n_neighbors=3)
knn.fit(X_train_2, y_train)
y_pred_2=knn.predict(X_test_2)


knn = neighbors.KNeighborsClassifier(n_neighbors=3)
X_train_3=X_train[:,range(4)]
X_test_3=X_test[:,range(4)]
knn.fit(X_train_3, y_train)
y_pred_3=knn.predict(X_test_3)

print("\n")
print("First five rows of the 4 features generated by taking first 4 features")
print(pd.DataFrame(X_train_3).head(5))

print("\n")
print(f"Accuracy of feature engineering: {accuracy_score(y_test, y_pred_1):.3f}")
print(f"Accuracy of PCA: {accuracy_score(y_test, y_pred_2):.3f}")
print(f"Accuracy of first four features: {accuracy_score(y_test, y_pred_3):.3f}")