Instructor: Kang-Pyo Lee
Nathan Ewing, Sung Chul Hong, Dong Yun Kim¶
Importing Modules¶
In [ ]:
# ! pip install --user --upgrade scikit-learn pyldavis
In [ ]:
# ! pip install --user wordcloud
In [1]:
import pandas as pd
import nltk
df2 = pd.read_csv("html_metadata.csv", sep = "\t")
df2 = df2.drop_duplicates(keep="first")
df2.article_title = df2.article_title.astype(str)
df2
Out[1]:
In [2]:
import nltk
import string
from nltk.corpus import stopwords
nltk.download('stopwords')
global_stopwords = stopwords.words("english")
local_stopwords = [c for c in string.punctuation] +\
['’', '``', '…', '...', "''", '‘', '“', '”', "'m", "'re", "'s", "'ve", 'amp', 'https', "n't", 'rt',
'a…', 'co', 'i…','it’s', 'â€\x9d', 'one', 'could','would', 'also', 'â€', 'said', '—', 'itâ']
In [3]:
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(use_idf=True, norm="l2", stop_words=global_stopwords+local_stopwords, max_df=0.7)
X = vectorizer.fit_transform(df2.article_title)
In [4]:
X.shape
Out[4]:
In [5]:
k = 5
from sklearn.cluster import KMeans
kmeans = KMeans(n_clusters=k, random_state=0)
kmeans
Out[5]:
In [6]:
%time kmeans.fit(X)
Out[6]:
In [7]:
kmeans.cluster_centers_
Out[7]:
In [8]:
kmeans.cluster_centers_.shape
Out[8]:
In [9]:
kmeans.labels_
Out[9]:
In [10]:
df2["cluster"] = kmeans.labels_
df2[["article_title", "cluster"]]
Out[10]:
In [11]:
df2.cluster.value_counts()
Out[11]:
In [12]:
counts = df2.cluster.value_counts()
df2[df2.cluster == counts.idxmax()].sample(10, replace=False, random_state=0)[["article_title", "cluster"]]
Out[12]:
In [13]:
df2[df2.cluster == counts.idxmin()].sample(10, replace=False, random_state=0)[["article_title", "cluster"]]
Out[13]:
In [14]:
import nltk
df2["words"] = df2.article_title.apply(lambda x: nltk.word_tokenize(x))
df2["tagged_words"] = df2.words.apply(lambda x: nltk.pos_tag(x))
from collections import Counter
def get_counter(dataframe, stopwords=[]):
counter = Counter()
for l in dataframe.tagged_words:
word_set = set()
for t in l:
word = t[0].lower()
tag = t[1]
if word not in stopwords:
word_set.add(word)
counter.update(word_set)
return counter
In [15]:
counter_max = get_counter(df2[df2.cluster == counts.idxmax()], global_stopwords+local_stopwords)
counter_max.most_common(30)
Out[15]:
In [16]:
counter_min = get_counter(df2[df2.cluster == counts.idxmin()], global_stopwords+local_stopwords)
counter_min.most_common(30)
Out[16]:
In [18]:
# ! pip install --user wordcloud
In [20]:
from wordcloud import WordCloud
from IPython.display import Image
wc = WordCloud(background_color="white", max_words=100, width=800, height=500)
wc.generate_from_frequencies(counter_max)
wc.to_file("wordcloud.png")
Image(filename="wordcloud.png")
Out[20]:
In [ ]:
### LDA Topic Modeling
In [21]:
num_topics = 5
In [22]:
from sklearn.decomposition import LatentDirichletAllocation as LDA
lda = LDA(n_components=num_topics, random_state=0) # LDA uses randomness to get a probability distribution
lda
Out[22]:
In [23]:
%time lda.fit(X)
Out[23]:
In [24]:
lda.components_
Out[24]:
In [25]:
lda.components_.shape
Out[25]:
In [26]:
def show_topics(model, feature_names, num_top_words):
for topic_idx, topic_scores in enumerate(model.components_):
print("***Topic {}:".format(topic_idx))
print(" + ".join(["{:.2f} * {}".format(topic_scores[i], feature_names[i]) for i in topic_scores.argsort()[::-1][:num_top_words]]))
print()
In [27]:
show_topics(lda, vectorizer.get_feature_names(), 10)
In [28]:
import pyLDAvis
import pyLDAvis.sklearn
pyLDAvis.enable_notebook()
In [29]:
pyLDAvis.sklearn.prepare(lda, X, vectorizer)
Out[29]:
In [ ]:
### End of Part 3