Exp_13_Document_recognition_tfidf_vectors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7e5225a7",
   "metadata": {},
   "source": [
    "# Document recogntion and text based search using tfidf vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18ae65a1",
   "metadata": {},
   "source": [
    "# 1. Dataset formulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d78f1f06",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "from sklearn.feature_extraction.text import TfidfTransformer \n",
    "from sklearn.feature_extraction.text import CountVectorizer \n",
    "docs=[\"the house had a tiny little mouse\", \n",
    "\"the cat saw the mouse\", \n",
    "\"the mouse ran away from the house\", \n",
    "\"the cat finally ate the mouse\", \n",
    "\"the end of the mouse story\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bff5e97c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.49356209, 0.39820278, 0.49356209, 0.23518498,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.23518498,\n",
       "        0.49356209],\n",
       "       [0.        , 0.        , 0.48334378, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.28547062,\n",
       "        0.        , 0.        , 0.59909216, 0.        , 0.57094124,\n",
       "        0.        ],\n",
       "       [0.        , 0.45709287, 0.        , 0.        , 0.        ,\n",
       "        0.45709287, 0.        , 0.36877965, 0.        , 0.2178072 ,\n",
       "        0.        , 0.45709287, 0.        , 0.        , 0.43561441,\n",
       "        0.        ],\n",
       "       [0.51392301, 0.        , 0.41462985, 0.        , 0.51392301,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.24488707,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.48977413,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.49175319, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.23432303,\n",
       "        0.49175319, 0.        , 0.        , 0.49175319, 0.46864606,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer \n",
    "# settings that you use for count vectorizer will go here \n",
    "tfidf_vectorizer=TfidfVectorizer(use_idf=True) \n",
    "# just send in all your docs here \n",
    "tfidf_vectorizer_vectors=tfidf_vectorizer.fit_transform(docs)\n",
    "type(tfidf_vectorizer_vectors)\n",
    "scipy.sparse.csr.csr_matrix.toarray(tfidf_vectorizer_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "35071daf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>had</th>\n",
       "      <td>0.493562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>little</th>\n",
       "      <td>0.493562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tiny</th>\n",
       "      <td>0.493562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>house</th>\n",
       "      <td>0.398203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mouse</th>\n",
       "      <td>0.235185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>0.235185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ate</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>away</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>finally</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>from</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>of</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ran</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>saw</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>story</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            tfidf\n",
       "had      0.493562\n",
       "little   0.493562\n",
       "tiny     0.493562\n",
       "house    0.398203\n",
       "mouse    0.235185\n",
       "the      0.235185\n",
       "ate      0.000000\n",
       "away     0.000000\n",
       "cat      0.000000\n",
       "end      0.000000\n",
       "finally  0.000000\n",
       "from     0.000000\n",
       "of       0.000000\n",
       "ran      0.000000\n",
       "saw      0.000000\n",
       "story    0.000000"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the first vector out (for the first document) \n",
    "first_vector_tfidfvectorizer=tfidf_vectorizer_vectors[0]\n",
    "# place tf-idf values in a pandas data frame \n",
    "df = pd.DataFrame(first_vector_tfidfvectorizer.T.todense(), \n",
    "                  index=tfidf_vectorizer.get_feature_names(), \n",
    "                  columns=[\"tfidf\"]) \n",
    "df.sort_values(by=[\"tfidf\"],ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d640757",
   "metadata": {},
   "source": [
    "##### Notice that the words ‘mouse’ and ‘the’ have the lowest IDF values. This is expected as these words appear in each and every document in our collection. The lower the IDF value of a word, the less unique it is to any particular document."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e799ca6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "words_to_find = ['away', 'house', 'mouse']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6d57614e",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer_tf_idf = TfidfVectorizer(max_df=.65, min_df=1, stop_words=None, \n",
    "                                    use_idf=True, norm=None, vocabulary=words_to_find)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5286b78a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sklearn.feature_extraction.text.TfidfVectorizer"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(vectorizer_tf_idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e9c33d17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(ngram_range=(1, 2), vocabulary=['away', 'house', 'mouse'])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_cnt = CountVectorizer(stop_words=None, vocabulary=words_to_find, ngram_range=(1,2))\n",
    "vectorizer_cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "df20d064",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = vectorizer_cnt.fit_transform(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "be8f97ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['away', 'house', 'mouse']\n",
      "[[0 1 1]\n",
      " [0 0 1]\n",
      " [1 1 1]\n",
      " [0 0 1]\n",
      " [0 0 1]]\n"
     ]
    }
   ],
   "source": [
    "print(vectorizer_cnt.get_feature_names())\n",
    "print(x.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "54b8c4f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.69314718, 0.        , 0.        ],\n",
       "       [0.        , 1.69314718, 0.        ],\n",
       "       [0.        , 0.        , 1.69314718]])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_vectorizer_vectors_1=vectorizer_tf_idf.fit_transform(words_to_find)\n",
    "type(tfidf_vectorizer_vectors_1)\n",
    "Xf = scipy.sparse.csr.csr_matrix.toarray(tfidf_vectorizer_vectors_1)\n",
    "Xf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7b768ae6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 16)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_vectorizer_vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "b04b12ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.49356209, 0.39820278, 0.49356209, 0.23518498,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.23518498,\n",
       "        0.49356209],\n",
       "       [0.        , 0.        , 0.48334378, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.28547062,\n",
       "        0.        , 0.        , 0.59909216, 0.        , 0.57094124,\n",
       "        0.        ],\n",
       "       [0.        , 0.45709287, 0.        , 0.        , 0.        ,\n",
       "        0.45709287, 0.        , 0.36877965, 0.        , 0.2178072 ,\n",
       "        0.        , 0.45709287, 0.        , 0.        , 0.43561441,\n",
       "        0.        ],\n",
       "       [0.51392301, 0.        , 0.41462985, 0.        , 0.51392301,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.24488707,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.48977413,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.49175319, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.23432303,\n",
       "        0.49175319, 0.        , 0.        , 0.49175319, 0.46864606,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xd = scipy.sparse.csr.csr_matrix.toarray(tfidf_vectorizer_vectors)\n",
    "Xd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "6d41ddf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6.23800253, 6.34842096, 6.22598832, 6.284547  , 6.28425291])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "single_point = Xf[0,0]\n",
    "points = Xd\n",
    "\n",
    "dist = (points - single_point)**2\n",
    "dist = np.sum(dist, axis=1)\n",
    "dist = np.sqrt(dist)\n",
    "dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "ab216879",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.6931471805599454"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xf[0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7077aac",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}