Exp_15_Zipfs_law_Test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "afeeb71e",
   "metadata": {},
   "source": [
    "# Test Zipf's LAW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b39b1fc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "D1 = \"\"\" A cat is a good pet and they are friendly with a cat you can ply in the sun and moon\n",
    "        will be there whn to play with cat the pepople are feel to have cat around \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "87fbb94b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#D2 = \"\"\" Oh my god dog is dog an dog off dog that bites with large group of my age is 10 and i ride \n",
    "#         bikes in the morning and evening go to bed in the night at hostel after dinner \"\"\"\n",
    "D2 = \"\"\"Oh my god dog dog dog dog that bites with large group of my age is 10 and i ride \n",
    "         bikes in the morning and evening go to bed in the night at hostel after dinner\"\"\"\n",
    "#D2 = \"\"\" Oh my dog god an bites with large group of my age dog is 10 and i ride \n",
    "#         bikes in the morning and dog evening go to bed in the night at hostel after dinner dog \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e8008f60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['A', 'cat', 'is', 'a', 'good', 'pet', 'and', 'they', 'are', 'friendly', 'with', 'a', 'cat', 'you', 'can', 'ply', 'in', 'the', 'sun', 'and', 'moon', 'will', 'be', 'there', 'whn', 'to', 'play', 'with', 'cat', 'the', 'pepople', 'are', 'feel', 'to', 'have', 'cat', 'around']\n",
      "['Oh', 'my', 'god', 'dog', 'dog', 'dog', 'dog', 'that', 'bites', 'with', 'large', 'group', 'of', 'my', 'age', 'is', '10', 'and', 'i', 'ride', 'bikes', 'in', 'the', 'morning', 'and', 'evening', 'go', 'to', 'bed', 'in', 'the', 'night', 'at', 'hostel', 'after', 'dinner']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "tokenizer = TreebankWordTokenizer()\n",
    "tokens_D1 = tokenizer.tokenize(D1)\n",
    "print(tokens_D1)\n",
    "tokens_D2 = tokenizer.tokenize(D2)\n",
    "print(tokens_D2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "433427d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'cat': 4, 'a': 2, 'and': 2, 'are': 2, 'with': 2, 'the': 2, 'to': 2, 'A': 1, 'is': 1, 'good': 1, 'pet': 1, 'they': 1, 'friendly': 1, 'you': 1, 'can': 1, 'ply': 1, 'in': 1, 'sun': 1, 'moon': 1, 'will': 1, 'be': 1, 'there': 1, 'whn': 1, 'play': 1, 'pepople': 1, 'feel': 1, 'have': 1, 'around': 1})\n",
      "Counter({'dog': 4, 'my': 2, 'and': 2, 'in': 2, 'the': 2, 'Oh': 1, 'god': 1, 'that': 1, 'bites': 1, 'with': 1, 'large': 1, 'group': 1, 'of': 1, 'age': 1, 'is': 1, '10': 1, 'i': 1, 'ride': 1, 'bikes': 1, 'morning': 1, 'evening': 1, 'go': 1, 'to': 1, 'bed': 1, 'night': 1, 'at': 1, 'hostel': 1, 'after': 1, 'dinner': 1})\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "token_counts_D1 = Counter(tokens_D1)\n",
    "print(token_counts_D1)\n",
    "token_counts_D2 = Counter(tokens_D2)\n",
    "print(token_counts_D2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e1605fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "import pandas as pd\n",
    "frequency = token_counts_D1\n",
    "rank = 1\n",
    "column_header = ['Rank', 'Frequency']\n",
    "df_D1 = pd.DataFrame( columns = column_header )\n",
    "collection = sorted(frequency.items(), key=itemgetter(1), reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7c3003e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         Rank Frequency\n",
      "cat         1         4\n",
      "a           2         2\n",
      "and         3         2\n",
      "are         4         2\n",
      "with        5         2\n",
      "the         6         2\n",
      "to          7         2\n",
      "A           8         1\n",
      "is          9         1\n",
      "good       10         1\n",
      "pet        11         1\n",
      "they       12         1\n",
      "friendly   13         1\n",
      "you        14         1\n",
      "can        15         1\n",
      "ply        16         1\n",
      "in         17         1\n",
      "sun        18         1\n",
      "moon       19         1\n",
      "will       20         1\n"
     ]
    }
   ],
   "source": [
    "for word , freq in token_counts_D1.most_common(20):\n",
    "    df_D1.loc[word] = [rank, freq]\n",
    "    rank = rank + 1\n",
    "print (df_D1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aaf32894",
   "metadata": {},
   "outputs": [],
   "source": [
    "frequency = token_counts_D2\n",
    "rank = 1\n",
    "column_header = ['Rank', 'Frequency']\n",
    "df_D2 = pd.DataFrame( columns = column_header )\n",
    "collection = sorted(frequency.items(), key=itemgetter(1), reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "962ed9a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Rank Frequency\n",
      "dog        1         4\n",
      "my         2         2\n",
      "and        3         2\n",
      "in         4         2\n",
      "the        5         2\n",
      "Oh         6         1\n",
      "god        7         1\n",
      "that       8         1\n",
      "bites      9         1\n",
      "with      10         1\n",
      "large     11         1\n",
      "group     12         1\n",
      "of        13         1\n",
      "age       14         1\n",
      "is        15         1\n",
      "10        16         1\n",
      "i         17         1\n",
      "ride      18         1\n",
      "bikes     19         1\n",
      "morning   20         1\n"
     ]
    }
   ],
   "source": [
    "for word , freq in token_counts_D2.most_common(20):\n",
    "    df_D2.loc[word] = [rank, freq]\n",
    "    rank = rank + 1\n",
    "print (df_D2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dbea3c07",
   "metadata": {},
   "outputs": [],
   "source": [
    "corp = [D1,D2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b270318b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(corp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0595bb7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "listToStr = ' '.join([str(elem) for elem in corp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bdad7c00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['A', 'cat', 'is', 'a', 'good', 'pet', 'and', 'they', 'are', 'friendly', 'with', 'a', 'cat', 'you', 'can', 'ply', 'in', 'the', 'sun', 'and', 'moon', 'will', 'be', 'there', 'whn', 'to', 'play', 'with', 'cat', 'the', 'pepople', 'are', 'feel', 'to', 'have', 'cat', 'around', 'Oh', 'my', 'god', 'dog', 'dog', 'dog', 'dog', 'that', 'bites', 'with', 'large', 'group', 'of', 'my', 'age', 'is', '10', 'and', 'i', 'ride', 'bikes', 'in', 'the', 'morning', 'and', 'evening', 'go', 'to', 'bed', 'in', 'the', 'night', 'at', 'hostel', 'after', 'dinner']\n"
     ]
    }
   ],
   "source": [
    "tokens = tokenizer.tokenize(listToStr)\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f1b4bc62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'A': 1,\n",
       "         'cat': 4,\n",
       "         'is': 2,\n",
       "         'a': 2,\n",
       "         'good': 1,\n",
       "         'pet': 1,\n",
       "         'and': 4,\n",
       "         'they': 1,\n",
       "         'are': 2,\n",
       "         'friendly': 1,\n",
       "         'with': 3,\n",
       "         'you': 1,\n",
       "         'can': 1,\n",
       "         'ply': 1,\n",
       "         'in': 3,\n",
       "         'the': 4,\n",
       "         'sun': 1,\n",
       "         'moon': 1,\n",
       "         'will': 1,\n",
       "         'be': 1,\n",
       "         'there': 1,\n",
       "         'whn': 1,\n",
       "         'to': 3,\n",
       "         'play': 1,\n",
       "         'pepople': 1,\n",
       "         'feel': 1,\n",
       "         'have': 1,\n",
       "         'around': 1,\n",
       "         'Oh': 1,\n",
       "         'my': 2,\n",
       "         'god': 1,\n",
       "         'dog': 4,\n",
       "         'that': 1,\n",
       "         'bites': 1,\n",
       "         'large': 1,\n",
       "         'group': 1,\n",
       "         'of': 1,\n",
       "         'age': 1,\n",
       "         '10': 1,\n",
       "         'i': 1,\n",
       "         'ride': 1,\n",
       "         'bikes': 1,\n",
       "         'morning': 1,\n",
       "         'evening': 1,\n",
       "         'go': 1,\n",
       "         'bed': 1,\n",
       "         'night': 1,\n",
       "         'at': 1,\n",
       "         'hostel': 1,\n",
       "         'after': 1,\n",
       "         'dinner': 1})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_counts = Counter(tokens)\n",
    "token_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4a7b7628",
   "metadata": {},
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "import pandas as pd\n",
    "frequency = token_counts\n",
    "rank = 1\n",
    "column_header = ['Rank', 'Frequency']\n",
    "df = pd.DataFrame( columns = column_header )\n",
    "collection = sorted(frequency.items(), key=itemgetter(1), reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8e582ea9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         Rank Frequency\n",
      "cat         1         4\n",
      "and         2         4\n",
      "the         3         4\n",
      "dog         4         4\n",
      "with        5         3\n",
      "in          6         3\n",
      "to          7         3\n",
      "is          8         2\n",
      "a           9         2\n",
      "are        10         2\n",
      "my         11         2\n",
      "A          12         1\n",
      "good       13         1\n",
      "pet        14         1\n",
      "they       15         1\n",
      "friendly   16         1\n",
      "you        17         1\n",
      "can        18         1\n",
      "ply        19         1\n",
      "sun        20         1\n"
     ]
    }
   ],
   "source": [
    "for word , freq in token_counts.most_common(20):\n",
    "    df.loc[word] = [rank, freq]\n",
    "    rank = rank + 1\n",
    "print (df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3891beec",
   "metadata": {},
   "source": [
    "# Compute TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8ea8bba2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cat': 0.14285714285714285}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D1_tf = {}\n",
    "D1_tf['cat'] = token_counts_D1['cat']/len(token_counts_D1)\n",
    "D1_tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b2796703",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dog': 0.13793103448275862}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D2_tf = {}\n",
    "D2_tf['dog'] = token_counts_D2['dog']/len(token_counts_D2)\n",
    "D2_tf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bd1efe1",
   "metadata": {},
   "source": [
    "# IDF Cal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2212cde9",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_docs_containing_cat = 0\n",
    "for doc in [token_counts_D1, token_counts_D2]:\n",
    "    if 'cat' in doc:\n",
    "        num_docs_containing_cat +=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b6876ca2",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_docs_containing_dog = 0\n",
    "for doc in [token_counts_D1, token_counts_D2]:\n",
    "    if 'dog' in doc:\n",
    "        num_docs_containing_dog +=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "005f8da0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_of_docs = 2\n",
    "D1_idf = {}\n",
    "D2_idf = {}\n",
    "D1_idf['cat'] = num_of_docs/num_docs_containing_cat\n",
    "D1_idf['cat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0a1364d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D2_idf['dog'] = num_of_docs/num_docs_containing_dog\n",
    "D2_idf['dog']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1ea04b93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D1_idf['dog'] = num_of_docs/num_docs_containing_dog\n",
    "D1_idf['dog']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dfec002",
   "metadata": {},
   "source": [
    "# TF_IDF of dog vs cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9ab75d4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# CAT\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "36af109f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2857142857142857"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TF_IDF_Cat = D1_tf['cat']*D1_idf['cat']\n",
    "TF_IDF_Cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b74d08c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.27586206896551724"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TF_IDF_dog = D2_tf['dog']*D2_idf['dog']\n",
    "TF_IDF_dog"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b483024b",
   "metadata": {},
   "source": [
    "# What do you think Happened here?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a9423a80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.5440680443502756, -0.5593080109070124)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math\n",
    "math.log(TF_IDF_Cat,10),math.log(TF_IDF_dog,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6bc621af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.04300428509485445, 0.04152137871227326)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TF_IDF_Cat = D1_tf['cat']*math.log(D1_idf['cat'],10)\n",
    "TF_IDF_dog = D2_tf['dog']*math.log(D2_idf['dog'],10)\n",
    "TF_IDF_Cat,TF_IDF_dog"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}