Exp_9_TFIDF_Vertorization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fee816d3",
   "metadata": {},
   "source": [
    "# Importance of TF IDF and usefulness of the vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "92f6bc19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = [\n",
    "...     'This is the first document',\n",
    "...     'This document is the second document',\n",
    "...     'And this is the third one',\n",
    "...     'Is this the first document',\n",
    "... ]\n",
    "type(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f0c3f5d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_listToStr = ' '.join([str(elem) for elem in corpus])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "aea228f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['this', 'is', 'the', 'first', 'document', 'this', 'document', 'is', 'the', 'second', 'document', 'and', 'this', 'is', 'the', 'third', 'one', 'is', 'this', 'the', 'first', 'document']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "tokenizer = TreebankWordTokenizer()\n",
    "tokens = tokenizer.tokenize(corpus_listToStr.lower())\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "58ee055e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "document_tfidf_vectors = []\n",
    "zero_vector = np.zeros([4, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "75b4ef96",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "for doc in corpus:\n",
    "    vec = copy.copy(zero_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "97f2df87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'this': 4,\n",
       "         'is': 4,\n",
       "         'the': 4,\n",
       "         'first': 2,\n",
       "         'document': 4,\n",
       "         'second': 1,\n",
       "         'and': 1,\n",
       "         'third': 1,\n",
       "         'one': 1})"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "token_counts = Counter(tokens)\n",
    "token_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "144895c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, value in token_counts.items():\n",
    "    docs_containing_key = 0\n",
    "    for _doc in corpus:\n",
    "        if key in _doc:\n",
    "            docs_containing_key += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "bbb6a241",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1111111111111111"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lexicon = sorted(set(tokens))\n",
    "tf = value/len(lexicon)\n",
    "tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c8fba0bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "if docs_containing_key:\n",
    "    idf = len(corpus)/docs_containing_key\n",
    "else:\n",
    "    idf = 0\n",
    "    vec[key] = tf*idf\n",
    "document_tfidf_vectors.append(vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "deaf9c7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.0"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf = len(corpus)/docs_containing_key\n",
    "idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b8f58cc7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0.]])]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_tfidf_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "6a067841",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "678475c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "3dabb716",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "7da59297",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse.csr.csr_matrix"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "acdf673b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.46979139, 0.58028582, 0.38408524, 0.        ,\n",
       "        0.        , 0.38408524, 0.        , 0.38408524],\n",
       "       [0.        , 0.6876236 , 0.        , 0.28108867, 0.        ,\n",
       "        0.53864762, 0.28108867, 0.        , 0.28108867],\n",
       "       [0.51184851, 0.        , 0.        , 0.26710379, 0.51184851,\n",
       "        0.        , 0.26710379, 0.51184851, 0.26710379],\n",
       "       [0.        , 0.46979139, 0.58028582, 0.38408524, 0.        ,\n",
       "        0.        , 0.38408524, 0.        , 0.38408524]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scipy\n",
    "scipy.sparse.csr.csr_matrix.toarray(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71f567df",
   "metadata": {},
   "source": [
    "### SEARCH A key word in corpus using TFIDF Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "6ca58715",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "# our corpus\n",
    "data = ['I like dog', 'I love cat', 'I interested in cat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "a6c23994",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "1bef7aa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x6 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 7 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert text data into term-frequency matrix\n",
    "data = cv.fit_transform(data)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "d678b3d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 0, 0, 1, 0],\n",
       "       [1, 0, 0, 0, 0, 1],\n",
       "       [1, 0, 1, 1, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scipy.sparse.csr.csr_matrix.toarray(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "3aa11ec3",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_transformer = TfidfTransformer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "2e7b9926",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x6 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 7 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert term-frequency matrix into tf-idf\n",
    "tfidf_matrix = tfidf_transformer.fit_transform(data)\n",
    "tfidf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "32a8bc74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.70710678, 0.        , 0.        , 0.70710678,\n",
       "        0.        ],\n",
       "       [0.60534851, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.79596054],\n",
       "       [0.4736296 , 0.        , 0.62276601, 0.62276601, 0.        ,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scipy.sparse.csr.csr_matrix.toarray(tfidf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "a37b8935",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cat': 1.2876820724517808,\n",
       " 'dog': 1.6931471805599454,\n",
       " 'in': 1.6931471805599454,\n",
       " 'interested': 1.6931471805599454,\n",
       " 'like': 1.6931471805599454,\n",
       " 'love': 1.6931471805599454}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create dictionary to find a tfidf word each word\n",
    "word2tfidf = dict(zip(cv.get_feature_names(), tfidf_transformer.idf_))\n",
    "word2tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "2aad327a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat 1.2876820724517808\n",
      "dog 1.6931471805599454\n",
      "in 1.6931471805599454\n",
      "interested 1.6931471805599454\n",
      "like 1.6931471805599454\n",
      "love 1.6931471805599454\n"
     ]
    }
   ],
   "source": [
    "for word, score in word2tfidf.items():\n",
    "    print(word, score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "7ae82a2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x2 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 2 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_text = ['I love cat']\n",
    "test_text = cv.fit_transform(test_text)\n",
    "tfidf_matrix1 = tfidf_transformer.fit_transform(test_text)\n",
    "tfidf_matrix1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "62b77b8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.70710678, 0.70710678]])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cc = scipy.sparse.csr.csr_matrix.toarray(tfidf_matrix1)\n",
    "cc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c2dfefd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}