topic modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "#import fasttext as ft\n",
    "import re\n",
    "import itertools\n",
    "import collections \n",
    "import nltk\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles = pd.read_csv('Tweets.csv')\n",
    "#articles = articles1.iloc[:100,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>airline_sentiment</th>\n",
       "      <th>airline_sentiment_confidence</th>\n",
       "      <th>negativereason</th>\n",
       "      <th>negativereason_confidence</th>\n",
       "      <th>airline</th>\n",
       "      <th>airline_sentiment_gold</th>\n",
       "      <th>name</th>\n",
       "      <th>negativereason_gold</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_coord</th>\n",
       "      <th>tweet_created</th>\n",
       "      <th>tweet_location</th>\n",
       "      <th>user_timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>570306133677760513</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cairdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica What @dhepburn said.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:35:52 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>570301130888122368</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.3486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:59 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>570301083672813571</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yvonnalynn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:48 -0800</td>\n",
       "      <td>Lets Play</td>\n",
       "      <td>Central Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>570301031407624196</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Bad Flight</td>\n",
       "      <td>0.7033</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:36 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>570300817074462722</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:45 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>570300767074181121</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>0.6842</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica seriously would pay $30 a fligh...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:33 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>570300616901320704</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.6745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cjmcginnis</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica yes, nearly every time I fly VX...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:13:57 -0800</td>\n",
       "      <td>San Francisco CA</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>570300248553349120</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6340</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pilot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica Really missed a prime opportuni...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:12:29 -0800</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>570299953286942721</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.6559</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dhepburn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@virginamerica Well, I didn't…but NOW I DO! :-D</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:11:19 -0800</td>\n",
       "      <td>San Diego</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>570295459631263746</td>\n",
       "      <td>positive</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>YupitsTate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it was amazing, and arrived an ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 10:53:27 -0800</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id airline_sentiment  airline_sentiment_confidence  \\\n",
       "0  570306133677760513           neutral                        1.0000   \n",
       "1  570301130888122368          positive                        0.3486   \n",
       "2  570301083672813571           neutral                        0.6837   \n",
       "3  570301031407624196          negative                        1.0000   \n",
       "4  570300817074462722          negative                        1.0000   \n",
       "5  570300767074181121          negative                        1.0000   \n",
       "6  570300616901320704          positive                        0.6745   \n",
       "7  570300248553349120           neutral                        0.6340   \n",
       "8  570299953286942721          positive                        0.6559   \n",
       "9  570295459631263746          positive                        1.0000   \n",
       "\n",
       "  negativereason  negativereason_confidence         airline  \\\n",
       "0            NaN                        NaN  Virgin America   \n",
       "1            NaN                     0.0000  Virgin America   \n",
       "2            NaN                        NaN  Virgin America   \n",
       "3     Bad Flight                     0.7033  Virgin America   \n",
       "4     Can't Tell                     1.0000  Virgin America   \n",
       "5     Can't Tell                     0.6842  Virgin America   \n",
       "6            NaN                     0.0000  Virgin America   \n",
       "7            NaN                        NaN  Virgin America   \n",
       "8            NaN                        NaN  Virgin America   \n",
       "9            NaN                        NaN  Virgin America   \n",
       "\n",
       "  airline_sentiment_gold        name negativereason_gold  retweet_count  \\\n",
       "0                    NaN     cairdin                 NaN              0   \n",
       "1                    NaN    jnardino                 NaN              0   \n",
       "2                    NaN  yvonnalynn                 NaN              0   \n",
       "3                    NaN    jnardino                 NaN              0   \n",
       "4                    NaN    jnardino                 NaN              0   \n",
       "5                    NaN    jnardino                 NaN              0   \n",
       "6                    NaN  cjmcginnis                 NaN              0   \n",
       "7                    NaN       pilot                 NaN              0   \n",
       "8                    NaN    dhepburn                 NaN              0   \n",
       "9                    NaN  YupitsTate                 NaN              0   \n",
       "\n",
       "                                                text tweet_coord  \\\n",
       "0                @VirginAmerica What @dhepburn said.         NaN   \n",
       "1  @VirginAmerica plus you've added commercials t...         NaN   \n",
       "2  @VirginAmerica I didn't today... Must mean I n...         NaN   \n",
       "3  @VirginAmerica it's really aggressive to blast...         NaN   \n",
       "4  @VirginAmerica and it's a really big bad thing...         NaN   \n",
       "5  @VirginAmerica seriously would pay $30 a fligh...         NaN   \n",
       "6  @VirginAmerica yes, nearly every time I fly VX...         NaN   \n",
       "7  @VirginAmerica Really missed a prime opportuni...         NaN   \n",
       "8    @virginamerica Well, I didn't…but NOW I DO! :-D         NaN   \n",
       "9  @VirginAmerica it was amazing, and arrived an ...         NaN   \n",
       "\n",
       "               tweet_created    tweet_location               user_timezone  \n",
       "0  2015-02-24 11:35:52 -0800               NaN  Eastern Time (US & Canada)  \n",
       "1  2015-02-24 11:15:59 -0800               NaN  Pacific Time (US & Canada)  \n",
       "2  2015-02-24 11:15:48 -0800         Lets Play  Central Time (US & Canada)  \n",
       "3  2015-02-24 11:15:36 -0800               NaN  Pacific Time (US & Canada)  \n",
       "4  2015-02-24 11:14:45 -0800               NaN  Pacific Time (US & Canada)  \n",
       "5  2015-02-24 11:14:33 -0800               NaN  Pacific Time (US & Canada)  \n",
       "6  2015-02-24 11:13:57 -0800  San Francisco CA  Pacific Time (US & Canada)  \n",
       "7  2015-02-24 11:12:29 -0800       Los Angeles  Pacific Time (US & Canada)  \n",
       "8  2015-02-24 11:11:19 -0800         San Diego  Pacific Time (US & Canada)  \n",
       "9  2015-02-24 10:53:27 -0800       Los Angeles  Eastern Time (US & Canada)  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topic_generation(x):\n",
    "    doc_complete = nltk.sent_tokenize(x.decode('utf-8')) # convert each training example to a list of documents\n",
    "    stop = set(nltk.corpus.stopwords.words('english')) #to remove stopwords\n",
    "    exclude = set(string.punctuation)                     #to remove punctuations\n",
    "    lemma = nltk.stem.wordnet.WordNetLemmatizer()       #to correct slang and return the base word\n",
    "    \n",
    "    def clean(doc):\n",
    "        stop_free = \" \".join([i for i in doc.lower().split() if i not in stop])\n",
    "        punc_free = ''.join(ch for ch in stop_free if ch not in exclude)\n",
    "        normalized = \" \".join(lemma.lemmatize(word) for word in punc_free.split())\n",
    "        return normalized                             \n",
    "\n",
    "    doc_clean = [clean(doc).split() for doc in doc_complete]   #list of docs where each doc contains its important words\n",
    "    dictionary = gensim.corpora.Dictionary(doc_clean)          \n",
    "\n",
    "    # Converting list of documents (corpus) into Document Term Matrix using dictionary prepared above.\n",
    "    doc_term_matrix = [dictionary.doc2bow(doc) for doc in doc_clean]\n",
    "    Lda = gensim.models.ldamodel.LdaModel\n",
    "\n",
    "    # Running and Trainign LDA model on the document term matrix.\n",
    "    try:\n",
    "        ldamodel = Lda(doc_term_matrix, num_topics=len(doc_complete), id2word = dictionary, passes=20)\n",
    "        topic_list = []\n",
    "\n",
    "        topics = ldamodel.show_topics(num_topics = len(doc_complete), num_words = 1) # num_words-no.of words under 1 topic(the one with the highest prob. for that topic)\n",
    "        for i in range(len(topics)):\n",
    "            if topics[i][1][7:-1] not in topic_list:\n",
    "                topic_list.append(topics[i][1][7:-1])\n",
    "        return topic_list\n",
    "\n",
    "    except ValueError:\n",
    "        pass\n",
    "   \n",
    "articles['topics'] = articles.text.map(lambda x:topic_generation(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                [dhepburn]\n",
       "1                                   [added]\n",
       "2                                 [another]\n",
       "3                              [aggressive]\n",
       "4                                     [bad]\n",
       "5                         [playing, flying]\n",
       "6                                    [away]\n",
       "7         [httpstcomwpg7grezp, opportunity]\n",
       "8                                 [well, d]\n",
       "9                             [early, good]\n",
       "10                                   [1024]\n",
       "11                [d, graphic, iconography]\n",
       "12                 [p, trip, virginamerica]\n",
       "13                         [fabulous, take]\n",
       "14                                 [thanks]\n",
       "15                                    [mia]\n",
       "16                      [cross, 29daystogo]\n",
       "17                              [help, nyc]\n",
       "18                     [virginamerica, ☺️👍]\n",
       "19                 [please, fly, amazingly]\n",
       "20                          [time, carrier]\n",
       "21       [virginamerica, httptcout5grrwaaa]\n",
       "22                             [love, feel]\n",
       "23                                [anytime]\n",
       "24                      [internet, seating]\n",
       "25              [applied, status, response]\n",
       "26                                  [2, ur]\n",
       "27                [virginamerica, together]\n",
       "28                         [sfotobos, cold]\n",
       "29                  [sendambien, red, noob]\n",
       "                        ...                \n",
       "14610              [outrageous, understand]\n",
       "14611                     [retribution, so]\n",
       "14612                [flight, help, anyway]\n",
       "14613                     [pleasehelp, got]\n",
       "14614             [backwards, calling, 8am]\n",
       "14615                         [sat, flight]\n",
       "14616                 [earlier, aa, flight]\n",
       "14617                    [americanair, big]\n",
       "14618            [luggage, seriously, 3078]\n",
       "14619                  [situation, company]\n",
       "14620                                 [1st]\n",
       "14621             [cant, help, americanair]\n",
       "14622                       [accommodation]\n",
       "14623                [nocharge, one, plane]\n",
       "14624                         [call, today]\n",
       "14625             [landing, 236, fantastic]\n",
       "14626            [nycbuenos, answer, going]\n",
       "14627          [americanair, flight, first]\n",
       "14628                       [review, thank]\n",
       "14629                         [americanair]\n",
       "14630                     [is, americanair]\n",
       "14631               [on, u, another, plane]\n",
       "14632                        [george, link]\n",
       "14633                              [flight]\n",
       "14634                         [americanair]\n",
       "14635                         [americanair]\n",
       "14636                    [called, late, 20]\n",
       "14637                             [airline]\n",
       "14638            [flight, make, suggestion]\n",
       "14639                           [4, flight]\n",
       "Name: topics, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles['topics']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop=time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "954.9847421646118"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop-start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}