Learning Topics in The Daily Kos with the Hierarchical Dirichlet Process¶

import itertools
import pyLDAvis
import pandas as pd
import re
import simplejson
import seaborn as sns

from microscopes.common.rng import rng
from microscopes.lda.definition import model_definition
from microscopes.lda.model import initialize
from microscopes.lda import model, runner
from random import shuffle
import matplotlib.pyplot as plt

sns.set_style('darkgrid')
sns.set_context('talk')

%matplotlib inline

!curl http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/docword.kos.txt.gz | gunzip > docword.kos.txt
!head docword.kos.txt

!curl http://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words/vocab.kos.txt > vocab.kos.txt
!head vocab.kos.txt

def parse_bag_of_words_file(docword, vocab):
    with open(vocab, "r") as f:
        kos_vocab = [word.strip() for word in f.readlines()]
        id_to_word = {i: word for i, word in enumerate(kos_vocab)}

    with open(docword, "r") as f:
        raw = [map(int, _.strip().split()) for _ in f.readlines()][3:]

    docs = []
    for _, grp in itertools.groupby(raw, lambda x: x[0]):
        doc = []
        for _, word_id, word_cnt in grp:
            doc += word_cnt * [id_to_word[word_id-1]]
        docs.append(doc)
    return docs, id_to_word

docs, id_to_word = parse_bag_of_words_file("docword.kos.txt", "vocab.kos.txt")
vocab_size = len(set(word for doc in docs for word in doc))

defn = model_definition(len(docs), vocab_size)
prng = rng()
kos_state = initialize(defn, docs, prng,
                        vocab_hp=1,
                        dish_hps={"alpha": 1, "gamma": 1})
r = runner.runner(defn, docs, kos_state)

print "number of docs:", defn.n, "vocabulary size:", defn.v

%%time
step_size = 50
steps = 500 / step_size

print "randomly initialized model:", "perplexity:", kos_state.perplexity(), "num topics:", kos_state.ntopics()
for s in range(steps):
    r.run(prng, step_size)
    print "iteration:", (s + 1) * step_size, "perplexity:", kos_state.perplexity(), "num topics:", kos_state.ntopics()

prepared = pyLDAvis.prepare(**kos_state.pyldavis_data())
pyLDAvis.display(prepared)

import cPickle as pickle

with open('kos_state.pkl','wb') as f:
    pickle.dump(kos_state, f)

with open('kos_state.pkl','rb') as f:
    new_state = pickle.load(f)

kos_state.assignments() == new_state.assignments()

kos_state.dish_assignments() == new_state.dish_assignments()

kos_state.table_assignments() == new_state.table_assignments()

kos_state = new_state

relevance = new_state.term_relevance_by_topic()

for topics in relevance:
    words = [word for word, _ in topics[:10]]
    print ' '.join(words)

doc7 = [word for word, _ in relevance[6][:100]]
shuffle(doc7)
doc_text = [word for word in doc7]
print ' '.join(doc_text)

predictions = pd.DataFrame()
predictions['From Topic 7'] = pd.Series(kos_state.predict(doc, rng())[0])

doc17 = [word for word, _ in relevance[0][:100]] + [word for word, _ in relevance[6][:100]]
shuffle(doc17)
predictions['From Topics 1 and 7']= pd.Series(kos_state.predict(doc17, r)[0])
predictions.plot(kind='bar')
plt.title('Prediced topic distribution')
plt.xticks(pred.index, ['%d' % (d+1) for d in xrange(len(pred.index))])
plt.xlabel('Topic Number')
plt.ylabel('Probability of Each Topic')

pd.Series(kos_state.topic_distribution_by_document()[0]).plot(kind='bar').set_title('Topic distribution for first document')
plt.xticks(pred.index, ['%d' % (d+1) for d in xrange(len(pred.index))])
plt.xlabel('Topic Number')
plt.ylabel('Probability of Each Topic')

pd.Series(kos_state.word_distribution_by_topic()[3]).sort(inplace=False).tail(10).plot(kind='barh')
plt.title('Top 10 Words in Topic 4')
plt.xlabel('Probability')