all

## THE following contains EVERYTHING !!!!!! 

Q1. Create a file “people.txt” with the following data: i) Read the data from the file “people.txt”. ii) Create a ruleset E that contain rules to check for the following conditions: 1. The age should be in the range 0-150. 2. The age should be greater than yearsmarried. 3. The status should be married or single or widowed. 4. If age is less than 18 the agegroup should be child, if age is between 18 and 65 the agegroup should be adult, if age is more than 65 the agegroup should be elderly. iii) Check whether ruleset E is violated by the data in the file people.txt. iv) Summarize the results obtained in part (iii) v) Visualize the results obtained in part (iii)
Code:
import pandas as pd import numpy as np # i)
df=pd.read_csv('people.txt',sep=',') df
# ii) def check_age():
return((df['Age']<150) & (df['Age']>0)) def check_years(): return(df['Age']>df['yearsmarried']) def check_status(): return((df['status']=='single') | (df['status']=='married') | (df['status']=='widowed')) def check_agegroup():
df.loc[df['Age']<18,'agegroup']='child' df.loc[(df['Age']>=18) & (df['Age']<65), 'agegroup']='adult'df.loc[df['Age']>=65,'agegroup']='elderly'
return(df)
# iii)
E={'Check Age':check_age(),'Check Years':check_years(),'check Status':
check_status(),'Check Agegroup': check_agegroup()}
E
# iv)
print('Age: ')
E['Check Age'].value_counts()
print('Years: ')
E['Check Years'].value_counts()
print('Status: ')
E['check Status'].value_counts()
# v)
import matplotlib.pyplot as plt
import seaborn as sns
plt.figure(figsize=(10,5))
plt.subplot(1,3,1)
E['Check Age'].value_counts().plot.bar()
plt.subplot(1,3,2)
E['Check Years'].value_counts().plot.bar()
plt.subplot(1,3,3)
E['check Status'].value_counts().plot.bar()


Q2. Perform the following preprocessing tasks on the dirty_iris datasetii.
i) Calculate the number and percentage of observations that are complete.
ii) Replace all the special values in data with NA.
iii) Define these rules in a separate text file and read them.(Use editfile function in R (package editrules). Use similar function in Python).
Print the resulting constraint object.
– Species should be one of the following values: setosa, versicolor or virginica.
– All measured numerical properties of an iris should be positive.
– The petal length of an iris is at least 2 times its petal width.
– The sepal length of an iris cannot exceed 30 cm.
– The sepals of an iris are longer than its petals.
iv)Determine how often each rule is broken (violatedEdits). Also summarize and
plot the
result.
v) Find outliers in sepal length using boxplot and boxplot.stats
Code:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
df = pd.read_csv("dirty_iris.txt")
df
#(i)No. and percentage of observation that are complete.
complete_obs = df.dropna()
num_complete_obs = len(complete_obs)
print(num_complete_obs)
print(num_complete_obs/len(df)*100)
#(ii) replace special value with NA
df1 = df.replace(np.nan,"NAN")
print(df1)
def checkSpecies():
return((df["Species"]=="versicolor")|(df["Species"]=="setosa")|(df["Species"]=="vir
ginica"))
def checkpositive():return((df["Sepal.Length"]>0)&(df["Sepal.Width"]>0)&(df["Petal.Length"]>0)&(df["
Petal.Width"]>0))
def checkplength():
return(df["Petal.Length"]>=df["Petal.Width"]*2)
def checkslength():
return(df["Sepal.Length"]<30)
def checkilength():
return(df["Sepal.Length"]>df["Petal.Length"])
RuleSet =
{"CheckSpecies":checkSpecies(),"Check+ve":checkpositive(),"CheckPlength":ch
eckplength(),"CheckSlength":checkslength(),"Checkilength":checkilength()}
pd.DataFrame(RuleSet).apply(pd.Series.value_counts).fillna(0)
pd.DataFrame(RuleSet).apply(pd.Series.value_counts).fillna(0).plot.bar()
import seaborn as sns
sns.boxplot(df['Sepal.Length'])
plt.show()


Q3. Load the data from wine dataset. Check whether all attributes are
standardized or not (mean
is 0 and standard deviation is 1). If not, standardize the attributes. Do the same
with Iris dataset.
Code:
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_wine, load_iriswine=load_wine()
data=wine.data
data.mean(axis=0)
data.std(axis=0)
st=StandardScaler()
data=st.fit_transform(data)
data.mean(axis=0)
data.std(axis=0)
iris=load_iris()
idata=iris.data
idata.mean(axis=0)
idata.std(axis=0)
idata=st.fit_transform(idata)
idata.mean(axis=0)
idata.std(axis=0)


Q4. Run Apriori algorithm to find frequent itemsets and association rules
1.1 Use minimum support as 50% and minimum confidence as 75%
1.2 Use minimum support as 60% and minimum confidence as 60 %
Code:
import numpy as np
import pandas as pd
from mlxtend.preprocessing import TransactionEncoder
from mlxtend.frequent_patterns import apriori, association_rulesdataset = [
['A', 'B', 'C', 'D', 'F', 'H'],
['B', 'E', 'F', 'H'],
['A', 'C', 'E'],
['B', 'C', 'D', 'F', 'H'],
['A', 'B', 'C', 'D', 'E'],
['C','D','F','H'],
['A','C','D','H'],
['E','H']
]
encoder = TransactionEncoder()
transactions = encoder.fit_transform(dataset)
data = pd.DataFrame(transactions, columns=encoder.columns_)
data
frequent_itemsets = apriori(data, min_support=0.5, use_colnames=True)
frequent_itemsets
association_rules(frequent_itemsets, metric='confidence', min_threshold=0.75)
frequent_itemsets = apriori(data, min_support=0.6, use_colnames=True)
frequent_itemsets
association_rules(frequent_itemsets, metric='confidence', min_threshold=0.6)


#Q5. Use Naive bayes, K-nearest, and Decision tree classification algorithms and build classifiers.Divide the data set into training and test set. Compare the accuracy of thedifferent classifiers under the following situations:
5.1 a) Training set = 75% Test set = 25% b) Training set = 66.6% (2/3rd of total),
Test set =
33.3%5.2 Training set is chosen by i) hold out method ii) Random subsampling iii)
Cross-Validation.
Compare the accuracy of the classifiers obtained.
5.3 Data is scaled to standard format.
Code:
from sklearn import datasets
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
cancer=datasets.load_breast_cancer()
## 5.1
### a) Training set = 75%, Test set = 25%
#### for decision tree
X_train,X_test,y_train,y_test=train_test_split(cancer.data,cancer.target,test_size=
0.25,random_state=0)
d_model=DecisionTreeClassifier(criterion='entropy',max_depth=None)
d_model.fit(X_train,y_train)
y_pred=d_model.predict(X_test)
a1=accuracy_score(y_test,y_pred)
print("Accuracy: ", a1*100)
#### for Naive Bayes
n_model=GaussianNB()
n_model.fit(X_train,y_train)y_predNaive=n_model.predict(X_test)
a2=accuracy_score(y_test,y_predNaive)
print('Accuracy: ', a2*100)
k_model=KNeighborsClassifier(5)
k_model.fit(X_train,y_train)
y_predK=k_model.predict(X_test)
ak=accuracy_score(y_test,y_predK)
print('Accuracy: ',ak*100)
### b) Training set = 66.6%, Test set = 33.3%
#### for decision tree
X1_train,X1_test,y1_train,y1_test=train_test_split(cancer.data,cancer.target,test_
size=0.33,random_state=0)
d_model.fit(X1_train,y1_train)
y1_pred=d_model.predict(X1_test)
a3=accuracy_score(y1_test,y1_pred)
print('Accuracy: ',a3*100)
#### for Naive Bayes
n_model.fit(X1_train,y1_train)
y1_predNaive=n_model.predict(X1_test)
a4=accuracy_score(y1_test,y1_predNaive)
print('Accuracy: ', a4*100)
k_model.fit(X1_train,y1_train)
y1_predk=k_model.predict(X1_test)print('Accuracy: ', accuracy_score(y1_test,y1_predk)*100)
## 5.2
### i) choosing training dataset from hold out method
#### for decision tree
X2_train,X2_test,y2_train,y2_test=train_test_split(cancer.data,cancer.target,test_
size=0.3,random_state=0)
d_model.fit(X2_train,y2_train)
y2_pred=d_model.predict(X2_test)
a6= accuracy_score(y2_test,y2_pred)
print('Accuracy: ', a6*100)
#### for Naive Bayes
n_model.fit(X2_train,y2_train)
y2_predNaive=n_model.predict(X2_test)
a7=accuracy_score(y2_test,y2_predNaive)
print('Accuracy: ', a7*100)
k_model.fit(X2_train,y2_train)
y2_predk=k_model.predict(X2_test)
print('Accuracy: ', accuracy_score(y2_test,y2_predk)*100)
### ii) random subsampling
#### for decision tree
a=[]
for i in range(5):
X3_train,X3_test,y3_train,y3_test=train_test_split(cancer.data,cancer.target,test_
size=0.3,random_state=0)d_model.fit(X3_train,y3_train)
y3_pred=d_model.predict(X3_test)
accuracy=accuracy_score(y3_test,y3_pred)
a.append(accuracy)
import numpy as np
avg_accuracy=np.mean(a)
print('Accuracy: ',avg_accuracy*100)
#### for Naive Bayes
a1=[]
for i in range(5):
X3_train,X3_test,y3_train,y3_test=train_test_split(cancer.data,cancer.target,test_
size=0.3,random_state=0)
n_model.fit(X3_train,y3_train)
y3_pred=n_model.predict(X3_test)
accuracy=accuracy_score(y3_test,y3_pred)
a1.append(accuracy)
avg_accuracy1=np.mean(a1)
print('Accuracy: ',avg_accuracy1*100)
a2=[]
for i in range(5):
X3_train,X3_test,y3_train,y3_test=train_test_split(cancer.data,cancer.target,test_
size=0.3,random_state=0)
k_model.fit(X3_train,y3_train)
y3_pred=k_model.predict(X3_test)
accuracy=accuracy_score(y3_test,y3_pred)
a2.append(accuracy)
avg_accuracyk=np.mean(a2)
print('Accuracy: ',avg_accuracyk*100)
### iii) cross-validation#### for decision tree
scores=cross_val_score(d_model,cancer.data,cancer.target,cv=10,scoring='accu
racy')
scores
print('Accuracy: ', scores.mean()*100)
#### for Naive Bayes
scores1=cross_val_score(n_model,cancer.data,cancer.target,cv=10,scoring='acc
uracy')
scores1
print('Accuracy: ', scores1.mean()*100)
scores2=cross_val_score(k_model,cancer.data,cancer.target,cv=10,scoring='acc
uracy')
scores2
print('Accuracy: ', scores2.mean()*100)


#Q6. Use Simple Kmeans, DBScan, Hierachical clustering algorithms for clustering. Compare the performance of clusters by changing the parameters involved in the algorithms.

#Code:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.cluster import KMeans, AgglomerativeClustering, DBSCANdata = load_iris(as_frame=True).frame
data.head()
### Plotting Sepal Width and Petal Length
for index in range(150):
if index <= 49:
plt.plot(data.values[index:, 1], data.values[index:, 2], 'ro')
elif index > 49 and index <= 99:
plt.plot(data.values[index:, 1], data.values[index:, 2], 'bo')
elif index > 99:
plt.plot(data.values[index:, 1], data.values[index:, 2], 'go')
plt.show()
## K-Means Clustering
k_cluster = KMeans(n_clusters=3)
k_cluster.fit(data.values[:, 1:3])
for index in range(150):
if k_cluster.labels_[index] == 0:
plt.plot(data.values[index:, 1], data.values[index:, 2], 'go')
elif k_cluster.labels_[index] == 1:
plt.plot(data.values[index:, 1], data.values[index:, 2], 'ro')
elif k_cluster.labels_[index] == 2:
plt.plot(data.values[index:, 1], data.values[index:, 2], 'bo')
plt.plot(k_cluster.cluster_centers_[:, 0], k_cluster.cluster_centers_[:, 1], 'o',
c='black')
plt.show()
## Hierarchical Agglomerative Clustering
agg_cluster = AgglomerativeClustering(n_clusters=3)agg_cluster.fit(data.values[:, 1:3])
for index in range(150): if agg_cluster.labels_[index] == 0: plt.plot(data.values[index:, 1], data.values[index:, 2], 'go')
elif agg_cluster.labels_[index] == 1: plt.plot(data.values[index:, 1], data.values[index:, 2], 'ro')
elif agg_cluster.labels_[index] == 2: plt.plot(data.values[index:, 1], data.values[index:, 2], 'bo')
plt.show() ## DBSCAN Clustering db_cluster = DBSCAN() db_cluster.fit(data.values[:, 1:3])
for index in range(150): if db_cluster.labels_[index] == 0: plt.plot(data.values[index:, 1], data.values[index:, 2], 'go')
elif db_cluster.labels_[index] == 1: plt.plot(data.values[index:, 1], data.values[index:, 2], 'ro')
elif db_cluster.labels_[index] == 2: plt.plot(data.values[index:, 1], data.values[index:, 2], 'bo')
plt.show()