EXP_7_Text_Sequence_Generator_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sequence to Sequense Prediction of Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.504353Z",
     "start_time": "2023-07-31T13:47:32.568367Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM, Dense, Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.520354Z",
     "start_time": "2023-07-31T13:47:43.505355Z"
    }
   },
   "outputs": [],
   "source": [
    "t_data = 'Jack and Jill went up the hill To fetch a pail of water Jack fell down and broke his crown, And Jill came tumbling after.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.569356Z",
     "start_time": "2023-07-31T13:47:43.520354Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Jack and Jill went up the hill To fetch a pail of water Jack fell down and broke his crown, And Jill came tumbling after.'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenization and Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.586359Z",
     "start_time": "2023-07-31T13:47:43.538356Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = tf.keras.preprocessing.text.Tokenizer()\n",
    "tokenizer.fit_on_texts([t_data])\n",
    "total_words = len(tokenizer.word_index) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.589355Z",
     "start_time": "2023-07-31T13:47:43.553360Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'and': 1, 'jack': 2, 'jill': 3, 'went': 4, 'up': 5, 'the': 6, 'hill': 7, 'to': 8, 'fetch': 9, 'a': 10, 'pail': 11, 'of': 12, 'water': 13, 'fell': 14, 'down': 15, 'broke': 16, 'his': 17, 'crown': 18, 'came': 19, 'tumbling': 20, 'after': 21}\n"
     ]
    }
   ],
   "source": [
    "word_ids = tokenizer.word_index\n",
    "print(word_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert the text into sequences for training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.590356Z",
     "start_time": "2023-07-31T13:47:43.568354Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16, 17, 18, 1, 3, 19, 20, 21]\n",
      "25\n",
      "[]\n",
      "0\n",
      "[[2, 1], [2, 1, 3], [2, 1, 3, 4], [2, 1, 3, 4, 5], [2, 1, 3, 4, 5, 6], [2, 1, 3, 4, 5, 6, 7], [2, 1, 3, 4, 5, 6, 7, 8], [2, 1, 3, 4, 5, 6, 7, 8, 9], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16, 17], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16, 17, 18], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16, 17, 18, 1], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16, 17, 18, 1, 3], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16, 17, 18, 1, 3, 19], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16, 17, 18, 1, 3, 19, 20], [2, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 2, 14, 15, 1, 16, 17, 18, 1, 3, 19, 20, 21]]\n"
     ]
    }
   ],
   "source": [
    "input_seq = []\n",
    "for line in t_data.split(\".\"):\n",
    "    token_list = tokenizer.texts_to_sequences([line])[0]\n",
    "    print(token_list)\n",
    "    print(len(token_list))\n",
    "    for i in range(1, len(token_list)):\n",
    "        num_of_words_per_seq = token_list[:i+1]\n",
    "        input_seq.append(num_of_words_per_seq)\n",
    "print(input_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.723355Z",
     "start_time": "2023-07-31T13:47:43.584355Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    }
   ],
   "source": [
    "print(len(input_seq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.738353Z",
     "start_time": "2023-07-31T13:47:43.599355Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maximum_length_of_seq = max(len(seq) for seq in input_seq)\n",
    "maximum_length_of_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.739354Z",
     "start_time": "2023-07-31T13:47:43.618356Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2\n",
      "   1]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1\n",
      "   3]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3\n",
      "   4]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4\n",
      "   5]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5\n",
      "   6]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6\n",
      "   7]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7\n",
      "   8]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8\n",
      "   9]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9\n",
      "  10]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10\n",
      "  11]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11\n",
      "  12]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12\n",
      "  13]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13\n",
      "   2]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2\n",
      "  14]\n",
      " [ 0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14\n",
      "  15]\n",
      " [ 0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15\n",
      "   1]\n",
      " [ 0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1\n",
      "  16]\n",
      " [ 0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16\n",
      "  17]\n",
      " [ 0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17\n",
      "  18]\n",
      " [ 0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18\n",
      "   1]\n",
      " [ 0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18  1\n",
      "   3]\n",
      " [ 0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18  1  3\n",
      "  19]\n",
      " [ 0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18  1  3 19\n",
      "  20]\n",
      " [ 2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18  1  3 19 20\n",
      "  21]]\n"
     ]
    }
   ],
   "source": [
    "input_seq_padded = tf.keras.preprocessing.sequence.pad_sequences(input_seq, maxlen=maximum_length_of_seq, padding='pre')\n",
    "print(input_seq_padded)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Construct the training and testing DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.740353Z",
     "start_time": "2023-07-31T13:47:43.631357Z"
    }
   },
   "outputs": [],
   "source": [
    "X, y = input_seq_padded[:, :-1], input_seq_padded[:, -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.740353Z",
     "start_time": "2023-07-31T13:47:43.646392Z"
    }
   },
   "outputs": [],
   "source": [
    "y = tf.keras.utils.to_categorical(y, num_classes=total_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.741354Z",
     "start_time": "2023-07-31T13:47:43.662355Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13]\n",
      " [ 0  0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2]\n",
      " [ 0  0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14]\n",
      " [ 0  0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15]\n",
      " [ 0  0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1]\n",
      " [ 0  0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16]\n",
      " [ 0  0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17]\n",
      " [ 0  0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18]\n",
      " [ 0  0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18  1]\n",
      " [ 0  0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18  1  3]\n",
      " [ 0  2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18  1  3 19]\n",
      " [ 2  1  3  4  5  6  7  8  9 10 11 12 13  2 14 15  1 16 17 18  1  3 19 20]]\n"
     ]
    }
   ],
   "source": [
    "print(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:43.742354Z",
     "start_time": "2023-07-31T13:47:43.678355Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]]\n"
     ]
    }
   ],
   "source": [
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM Model Building using Tensorflow Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:44.194187Z",
     "start_time": "2023-07-31T13:47:43.695355Z"
    }
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(total_words, 50,input_length=maximum_length_of_seq-1))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dense(total_words, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:44.213774Z",
     "start_time": "2023-07-31T13:47:44.196157Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 24, 50)            1100      \n",
      "                                                                 \n",
      " lstm (LSTM)                 (None, 100)               60400     \n",
      "                                                                 \n",
      " dense (Dense)               (None, 22)                2222      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 63,722\n",
      "Trainable params: 63,722\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compile the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:44.251805Z",
     "start_time": "2023-07-31T13:47:44.214776Z"
    }
   },
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:47.957592Z",
     "start_time": "2023-07-31T13:47:44.243774Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "1/1 [==============================] - 2s 2s/step - loss: 3.0648 - accuracy: 0.1250\n",
      "Epoch 2/100\n",
      "1/1 [==============================] - 0s 21ms/step - loss: 3.0590 - accuracy: 0.1250\n",
      "Epoch 3/100\n",
      "1/1 [==============================] - 0s 21ms/step - loss: 3.0526 - accuracy: 0.1250\n",
      "Epoch 4/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 3.0454 - accuracy: 0.1250\n",
      "Epoch 5/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 3.0372 - accuracy: 0.1250\n",
      "Epoch 6/100\n",
      "1/1 [==============================] - 0s 21ms/step - loss: 3.0275 - accuracy: 0.1250\n",
      "Epoch 7/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 3.0160 - accuracy: 0.1250\n",
      "Epoch 8/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 3.0027 - accuracy: 0.1250\n",
      "Epoch 9/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 2.9888 - accuracy: 0.1250\n",
      "Epoch 10/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 2.9806 - accuracy: 0.1250\n",
      "Epoch 11/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 2.9823 - accuracy: 0.1250\n",
      "Epoch 12/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.9724 - accuracy: 0.1250\n",
      "Epoch 13/100\n",
      "1/1 [==============================] - 0s 20ms/step - loss: 2.9585 - accuracy: 0.1250\n",
      "Epoch 14/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 2.9484 - accuracy: 0.1250\n",
      "Epoch 15/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.9409 - accuracy: 0.1250\n",
      "Epoch 16/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 2.9326 - accuracy: 0.1250\n",
      "Epoch 17/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.9215 - accuracy: 0.1250\n",
      "Epoch 18/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.9067 - accuracy: 0.1250\n",
      "Epoch 19/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.8875 - accuracy: 0.1250\n",
      "Epoch 20/100\n",
      "1/1 [==============================] - 0s 22ms/step - loss: 2.8639 - accuracy: 0.1250\n",
      "Epoch 21/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.8364 - accuracy: 0.1250\n",
      "Epoch 22/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.8050 - accuracy: 0.1250\n",
      "Epoch 23/100\n",
      "1/1 [==============================] - 0s 20ms/step - loss: 2.7659 - accuracy: 0.1250\n",
      "Epoch 24/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.7171 - accuracy: 0.1250\n",
      "Epoch 25/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.6634 - accuracy: 0.1667\n",
      "Epoch 26/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 2.6103 - accuracy: 0.1667\n",
      "Epoch 27/100\n",
      "1/1 [==============================] - 0s 14ms/step - loss: 2.5603 - accuracy: 0.1250\n",
      "Epoch 28/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.5122 - accuracy: 0.1250\n",
      "Epoch 29/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.4648 - accuracy: 0.1667\n",
      "Epoch 30/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.4162 - accuracy: 0.1250\n",
      "Epoch 31/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 2.3651 - accuracy: 0.2083\n",
      "Epoch 32/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 2.3119 - accuracy: 0.3333\n",
      "Epoch 33/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.2592 - accuracy: 0.2083\n",
      "Epoch 34/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.2473 - accuracy: 0.2917\n",
      "Epoch 35/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 2.2714 - accuracy: 0.2500\n",
      "Epoch 36/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.2267 - accuracy: 0.2500\n",
      "Epoch 37/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.1132 - accuracy: 0.2500\n",
      "Epoch 38/100\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 2.2065 - accuracy: 0.2083\n",
      "Epoch 39/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 2.0606 - accuracy: 0.2917\n",
      "Epoch 40/100\n",
      "1/1 [==============================] - 0s 14ms/step - loss: 2.0751 - accuracy: 0.4167\n",
      "Epoch 41/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.0726 - accuracy: 0.3333\n",
      "Epoch 42/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 1.9952 - accuracy: 0.4167\n",
      "Epoch 43/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.9537 - accuracy: 0.5000\n",
      "Epoch 44/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.9718 - accuracy: 0.2917\n",
      "Epoch 45/100\n",
      "1/1 [==============================] - 0s 20ms/step - loss: 1.9007 - accuracy: 0.3750\n",
      "Epoch 46/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.8719 - accuracy: 0.4167\n",
      "Epoch 47/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.8601 - accuracy: 0.3333\n",
      "Epoch 48/100\n",
      "1/1 [==============================] - 0s 20ms/step - loss: 1.8120 - accuracy: 0.4583\n",
      "Epoch 49/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.7917 - accuracy: 0.4583\n",
      "Epoch 50/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.7532 - accuracy: 0.5417\n",
      "Epoch 51/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 1.7116 - accuracy: 0.5833\n",
      "Epoch 52/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.7080 - accuracy: 0.5833\n",
      "Epoch 53/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.6568 - accuracy: 0.7083\n",
      "Epoch 54/100\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 1.6304 - accuracy: 0.6667\n",
      "Epoch 55/100\n",
      "1/1 [==============================] - 0s 14ms/step - loss: 1.6059 - accuracy: 0.7500\n",
      "Epoch 56/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 1.5704 - accuracy: 0.8333\n",
      "Epoch 57/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.5466 - accuracy: 0.7083\n",
      "Epoch 58/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.5077 - accuracy: 0.7500\n",
      "Epoch 59/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 1.4982 - accuracy: 0.7500\n",
      "Epoch 60/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.4605 - accuracy: 0.7500\n",
      "Epoch 61/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.4410 - accuracy: 0.7500\n",
      "Epoch 62/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.4135 - accuracy: 0.7917\n",
      "Epoch 63/100\n",
      "1/1 [==============================] - 0s 21ms/step - loss: 1.3866 - accuracy: 0.7500\n",
      "Epoch 64/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 1.3555 - accuracy: 0.7500\n",
      "Epoch 65/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.3404 - accuracy: 0.7917\n",
      "Epoch 66/100\n",
      "1/1 [==============================] - 0s 21ms/step - loss: 1.3055 - accuracy: 0.8333\n",
      "Epoch 67/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 1.2943 - accuracy: 0.8333\n",
      "Epoch 68/100\n",
      "1/1 [==============================] - 0s 14ms/step - loss: 1.2639 - accuracy: 0.9167\n",
      "Epoch 69/100\n",
      "1/1 [==============================] - 0s 21ms/step - loss: 1.2448 - accuracy: 0.8750\n",
      "Epoch 70/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 1.2258 - accuracy: 0.8333\n",
      "Epoch 71/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.2002 - accuracy: 0.8750\n",
      "Epoch 72/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 1.1782 - accuracy: 0.9167\n",
      "Epoch 73/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 1.1644 - accuracy: 0.8750\n",
      "Epoch 74/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.1370 - accuracy: 0.9167\n",
      "Epoch 75/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 1.1173 - accuracy: 0.9167\n",
      "Epoch 76/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.0986 - accuracy: 0.9167\n",
      "Epoch 77/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 1.0772 - accuracy: 0.9167\n",
      "Epoch 78/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.0643 - accuracy: 0.9167\n",
      "Epoch 79/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.0758 - accuracy: 0.8333\n",
      "Epoch 80/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 1.1229 - accuracy: 0.7917\n",
      "Epoch 81/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 1.0626 - accuracy: 0.7500\n",
      "Epoch 82/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.0547 - accuracy: 0.8333\n",
      "Epoch 83/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 18ms/step - loss: 1.0551 - accuracy: 0.8333\n",
      "Epoch 84/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 1.0353 - accuracy: 0.8750\n",
      "Epoch 85/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 0.9707 - accuracy: 0.9167\n",
      "Epoch 86/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 1.0135 - accuracy: 0.8333\n",
      "Epoch 87/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 0.9748 - accuracy: 0.9167\n",
      "Epoch 88/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 0.9808 - accuracy: 0.8333\n",
      "Epoch 89/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 0.9433 - accuracy: 0.9167\n",
      "Epoch 90/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 0.9241 - accuracy: 0.9583\n",
      "Epoch 91/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 0.9343 - accuracy: 0.9167\n",
      "Epoch 92/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 0.9061 - accuracy: 0.9167\n",
      "Epoch 93/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 0.8811 - accuracy: 1.0000\n",
      "Epoch 94/100\n",
      "1/1 [==============================] - 0s 14ms/step - loss: 0.8942 - accuracy: 0.9583\n",
      "Epoch 95/100\n",
      "1/1 [==============================] - 0s 14ms/step - loss: 0.8690 - accuracy: 0.9583\n",
      "Epoch 96/100\n",
      "1/1 [==============================] - 0s 20ms/step - loss: 0.8509 - accuracy: 0.9583\n",
      "Epoch 97/100\n",
      "1/1 [==============================] - 0s 18ms/step - loss: 0.8520 - accuracy: 0.9167\n",
      "Epoch 98/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 0.8396 - accuracy: 0.9167\n",
      "Epoch 99/100\n",
      "1/1 [==============================] - 0s 14ms/step - loss: 0.8212 - accuracy: 0.9167\n",
      "Epoch 100/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 0.8129 - accuracy: 0.9583\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1e52a7fded0>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X, y, epochs=100, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now Lets check how the model generates Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:47.996583Z",
     "start_time": "2023-07-31T13:47:47.952585Z"
    }
   },
   "outputs": [],
   "source": [
    "your_seed_text = \"You:Jack\"\n",
    "next_words = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:48.581518Z",
     "start_time": "2023-07-31T13:47:47.969587Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Jack and jill went up the hill to fetch a pail of water jack fell down and broke his crown and\n"
     ]
    }
   ],
   "source": [
    "for _ in range(next_words):\n",
    "    token_list = tokenizer.texts_to_sequences([your_seed_text])[0]\n",
    "    token_list = tf.keras.preprocessing.sequence.pad_sequences([token_list], maxlen=maximum_length_of_seq-1, padding='pre')\n",
    "    predicted = model.predict(token_list, verbose=0)\n",
    "    predicted_word_index = tf.argmax(predicted, axis=-1)\n",
    "    output_word = \"\"\n",
    "    for word, index in tokenizer.word_index.items():\n",
    "        if index == predicted_word_index:\n",
    "            output_word = word\n",
    "            break\n",
    "    your_seed_text += \" \" + output_word\n",
    "print(f\"Model: {your_seed_text.replace('You:', '')}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:48.633559Z",
     "start_time": "2023-07-31T13:47:48.583531Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You:Jack and jill went up the hill to fetch a pail of water jack fell down and broke his crown and\n"
     ]
    }
   ],
   "source": [
    "print(your_seed_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-31T13:47:48.634521Z",
     "start_time": "2023-07-31T13:47:48.598521Z"
    }
   },
   "outputs": [],
   "source": [
    "SimpleRNN"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}