In [40]:
import pandas as pd
from sklearn.cluster import DBSCAN
from sklearn import metrics
from sklearn.preprocessing import StandardScaler
import numpy as np
import matplotlib.pyplot as plt
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib import cm
from colorspacious import cspace_converter
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn.cluster import KMeans
from sklearn import preprocessing
from sklearn.cluster import KMeans
import seaborn as sns
import pandas as pd
import sklearn
from sklearn.cluster import KMeans
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
import matplotlib.pyplot as plt
## For Stemming
import os
import re ## for regular expressions
import matplotlib.cm as cm
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.cluster import KMeans
import sklearn
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.cluster import DBSCAN
from sklearn.cluster import AgglomerativeClustering
import scipy.cluster.hierarchy as hc
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import numpy as np
import pandas as pd
from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
In [18]:
My_Orig_DF=pd.read_csv("Labeled_Headline_Data_from_API.csv", sep=',', index_col=0)
In [19]:
SS_dist = []
values_for_k=range(2,20)
#print(values_for_k)DF_TF
for k_val in values_for_k:
print(k_val)
k_means = KMeans(n_clusters=k_val)
model = k_means.fit(DF_TF)
SS_dist.append(k_means.inertia_)
print(SS_dist)
print(values_for_k)
plt.plot(values_for_k, SS_dist, 'bx-')
plt.xlabel('value')
plt.ylabel('Sum of squared distances')
plt.title('Elbow method for optimal k Choice')
plt.show()
In [20]:
#####
# KMEANS
# Use k-means clustering on the data.
# Create clusters
k = 10
## Sklearn required you to instantiate first
kmeans = KMeans(n_clusters=k)
kmeans.fit(DF_TF) ## run kmeans
labels = kmeans.labels_
print(labels)
centroids = kmeans.cluster_centers_
print(centroids)
prediction = kmeans.predict(DF_TF)
print(prediction)
In [21]:
DF_TF_normalized=(DF_TF - DF_TF.mean()) / DF_TF.std()
print(DF_TF_normalized)
In [26]:
print(DF_TF_normalized.shape[0]) ## num rows
print(DF_TF_normalized.shape[1]) ## num cols
NumCols=DF_TF_normalized.shape[1]
# ## Instantiated my own copy of PCA
My_pca = PCA(n_components=2) ## I want the two prin columns
## Transpose it
DF_TF_normalized=np.transpose(DF_TF_normalized)
My_pca.fit(DF_TF_normalized)
Out[26]:
In [27]:
## Transpose it
DF_TF_normalized=np.transpose(DF_TF_normalized)
My_pca.fit(DF_TF_normalized)
print(My_pca)
print(My_pca.components_.T)
# KnownLabels=["solar", "nuclear", "fossil", "hydro" ]
# Reformat and view results
Comps = pd.DataFrame(My_pca.components_.T,
columns=['PC%s' % _ for _ in range(2)],
index=DF_TF_normalized.columns
)
print(Comps)
print(Comps.iloc[:,0])
RowNames = list(Comps.index)
KnownLabels=RowNames
#print(RowNames)
In [28]:
plt.figure(figsize=(12,12))
plt.scatter(Comps.iloc[:,0], Comps.iloc[:,1], s=100)#, color="green")
plt.xlabel("PC 1")
plt.ylabel("PC 2")
plt.title("Scatter Plot After Perfoming PCA",fontsize=15)
for i, label in enumerate(KnownLabels):
#print(i)
#print(label)
plt.annotate(label, (Comps.iloc[i,0], Comps.iloc[i,1]), fontsize=5)
plt.show()
In [29]:
type(Comps)
Comps.shape
from sklearn.cluster import KMeans
import numpy as np
k = 10 ##set the amount of clusters to be made
kmeans = KMeans(n_clusters=k) # initialize the class object
labels = kmeans.fit_predict(Comps) # predict the centers for all points
# print(labels)
labels_unique = np.unique(labels)
## plotting the results
colors = cm.rainbow(np.linspace(0, 1, k)) #make different colors for each cluster(here 10 colors are made)
Comps=np.array(Comps) # change to numpy array
centers = kmeans.cluster_centers_
for i, c in zip(labels_unique, colors):
plt.scatter(Comps[labels == i, 0], Comps[labels == i, 1], label = i, color=c)
for i, label in enumerate(KnownLabels):
plt.annotate(label, (Comps[i,0], Comps[i,1]), fontsize=5)
# plt.scatter(centers[:,0] , centers[:,1] , s = 80, color = 'k')
plt.legend(loc='upper right', bbox_to_anchor=(1.13, 1))
plt.title('KMeans Clustering with K=10')
plt.show()
In [30]:
#DBSCAN
##
###############################################
MyDBSCAN = DBSCAN(eps=0.1, min_samples=2)
## eps:
## The maximum distance between two samples for
##one to be considered as in the neighborhood of the other.
MyDBSCAN.fit_predict(DF_TF)
print(MyDBSCAN.labels_)
In [31]:
range_n_clusters = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
avg_list=[]
print(Comps)
for n_clusters in range_n_clusters:
# Create a subplot with 1 row and 2 columns
fig, (ax1, ax2) = plt.subplots(1, 2)
fig.set_size_inches(18, 7)
# The 1st subplot is the silhouette plot
# The silhouette coefficient can range from -1, 1 but in this example all
# lie within [-0.1, 1]
ax1.set_xlim([-0.1, 1])
# The (n_clusters+1)*10 is for inserting blank space between silhouette
# plots of individual clusters, to demarcate them clearly.
ax1.set_ylim([0, len(Comps) + (n_clusters + 1) * 10])
# Initialize the clusterer with n_clusters value and a random generator
# seed of 10 for reproducibility.
clusterer = KMeans(n_clusters=n_clusters, random_state=10)
cluster_labels = clusterer.fit_predict(Comps)
# The silhouette_score gives the average value for all the samples.
# This gives a perspective into the density and separation of the formed
# clusters
# avg_list=[]
silhouette_avg = silhouette_score(Comps, cluster_labels)
avg_list.append(silhouette_avg)
print("For n_clusters =", n_clusters,
"The average silhouette_score is :", silhouette_avg)
# Compute the silhouette scores for each sample
sample_silhouette_values = silhouette_samples(Comps, cluster_labels)
y_lower = 10
for i in range(n_clusters):
# Aggregate the silhouette scores for samples belonging to
# cluster i, and sort them
ith_cluster_silhouette_values = \
sample_silhouette_values[cluster_labels == i]
ith_cluster_silhouette_values.sort()
size_cluster_i = ith_cluster_silhouette_values.shape[0]
y_upper = y_lower + size_cluster_i
color = cm.nipy_spectral(float(i) / n_clusters)
ax1.fill_betweenx(np.arange(y_lower, y_upper),
0, ith_cluster_silhouette_values,
facecolor=color, edgecolor=color, alpha=0.7)
# Label the silhouette plots with their cluster numbers at the middle
ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
# Compute the new y_lower for next plot
y_lower = y_upper + 10 # 10 for the 0 samples
ax1.set_title("The silhouette plot for the various clusters.")
ax1.set_xlabel("The silhouette coefficient values")
ax1.set_ylabel("Cluster label")
# The vertical line for average silhouette score of all the values
ax1.axvline(x=silhouette_avg, color="red", linestyle="--")
ax1.set_yticks([]) # Clear the yaxis labels / ticks
ax1.set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])
# 2nd Plot showing the actual clusters formed
colors = cm.nipy_spectral(cluster_labels.astype(float) / n_clusters)
ax2.scatter(Comps[:, 0], Comps[:, 1], marker='.', s=30, lw=0, alpha=0.7,
c=colors, edgecolor='k')
# Labeling the clusters
centers = clusterer.cluster_centers_
# Draw white circles at cluster centers
ax2.scatter(centers[:, 0], centers[:, 1], marker='o',
c="white", alpha=1, s=200, edgecolor='k')
for i, c in enumerate(centers):
ax2.scatter(c[0], c[1], marker='$%d$' % i, alpha=1,
s=50, edgecolor='k')
ax2.set_title("The visualization of the clustered data.")
ax2.set_xlabel("Feature space for the 1st feature")
ax2.set_ylabel("Feature space for the 2nd feature")
plt.suptitle(("Silhouette analysis for KMeans clustering on sample data "
"with n_clusters = %d" % n_clusters),
fontsize=14, fontweight='bold')
plt.show()
In [32]:
# ## SILHOUETTE
# Make single plot of the silhouette scores
# =============================================================================
plt.plot(range_n_clusters, avg_list, 'bx-')
plt.xlabel("Number of Clusters")
plt.ylabel("Silhouette Score")
plt.title("Silhouette Plot",fontsize=15)
plt.show()
In [38]:
plt.figure(figsize =(12, 8))
plt.title('Hierarchical Clustering on NEWSAPI Text Data')
dendro = hc.dendrogram((hc.linkage(DF_TF, method ='ward')), leaf_font_size=7, labels=DF_TF.index)
plt.show()
In [ ]: