Learn practical skills, build real-world projects, and advance your career
Created 4 years ago
#@title Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, Bidirectional
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import regularizers
import tensorflow.keras.utils as ku
import numpy as np
tokenizer = Tokenizer()
!wget --no-check-certificate \
https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sonnets.txt \
-O /tmp/sonnets.txt
data = open('/tmp/sonnets.txt').read()
corpus = data.lower().split("\n")
tokenizer.fit_on_texts(corpus)
total_words = len(tokenizer.word_index) + 1
# create input sequences using list of tokens
input_sequences = []
for line in corpus:
token_list = tokenizer.texts_to_sequences([line])[0]
for i in range(1, len(token_list)):
n_gram_sequence = token_list[:i+1]
input_sequences.append(n_gram_sequence)
# pad sequences
max_sequence_len = max([len(x) for x in input_sequences])
input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))
# create predictors and label
predictors, label = input_sequences[:,:-1],input_sequences[:,-1]
label = ku.to_categorical(label, num_classes=total_words)
--2020-10-01 10:46:18-- https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sonnets.txt
Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.195.128, 74.125.142.128, 74.125.20.128, ...
Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.195.128|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 93578 (91K) [text/plain]
Saving to: ‘/tmp/sonnets.txt’
/tmp/sonnets.txt 100%[===================>] 91.38K --.-KB/s in 0.001s
2020-10-01 10:46:19 (86.4 MB/s) - ‘/tmp/sonnets.txt’ saved [93578/93578]
model = Sequential()
model.add(Embedding(total_words, 100, input_length=max_sequence_len-1))
model.add(Bidirectional(LSTM(150, return_sequences = True)))
model.add(Dropout(0.2))
model.add(LSTM(100))
model.add(Dense(total_words/2, activation='relu', kernel_regularizer=regularizers.l2(0.01)))
model.add(Dense(total_words, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 10, 100) 321100
_________________________________________________________________
bidirectional (Bidirectional (None, 10, 300) 301200
_________________________________________________________________
dropout (Dropout) (None, 10, 300) 0
_________________________________________________________________
lstm_1 (LSTM) (None, 100) 160400
_________________________________________________________________
dense (Dense) (None, 1605) 162105
_________________________________________________________________
dense_1 (Dense) (None, 3211) 5156866
=================================================================
Total params: 6,101,671
Trainable params: 6,101,671
Non-trainable params: 0
_________________________________________________________________
None
history = model.fit(predictors, label, epochs=100, verbose=1)
Epoch 1/100
484/484 [==============================] - 6s 11ms/step - loss: 6.9135 - accuracy: 0.0211
Epoch 2/100
484/484 [==============================] - 5s 11ms/step - loss: 6.5061 - accuracy: 0.0232
Epoch 3/100
484/484 [==============================] - 6s 12ms/step - loss: 6.4053 - accuracy: 0.0244
Epoch 4/100
484/484 [==============================] - 5s 11ms/step - loss: 6.2909 - accuracy: 0.0292
Epoch 5/100
484/484 [==============================] - 5s 11ms/step - loss: 6.1971 - accuracy: 0.0378
Epoch 6/100
484/484 [==============================] - 5s 11ms/step - loss: 6.1161 - accuracy: 0.0379
Epoch 7/100
484/484 [==============================] - 5s 11ms/step - loss: 6.0372 - accuracy: 0.0413
Epoch 8/100
484/484 [==============================] - 5s 11ms/step - loss: 5.9495 - accuracy: 0.0460
Epoch 9/100
484/484 [==============================] - 5s 11ms/step - loss: 5.8390 - accuracy: 0.0514
Epoch 10/100
484/484 [==============================] - 6s 11ms/step - loss: 5.7395 - accuracy: 0.0546
Epoch 11/100
484/484 [==============================] - 6s 12ms/step - loss: 5.6333 - accuracy: 0.0641
Epoch 12/100
484/484 [==============================] - 5s 11ms/step - loss: 5.5264 - accuracy: 0.0715
Epoch 13/100
484/484 [==============================] - 5s 11ms/step - loss: 5.4278 - accuracy: 0.0775
Epoch 14/100
484/484 [==============================] - 5s 11ms/step - loss: 5.3258 - accuracy: 0.0829
Epoch 15/100
484/484 [==============================] - 5s 11ms/step - loss: 5.2237 - accuracy: 0.0902
Epoch 16/100
484/484 [==============================] - 6s 12ms/step - loss: 5.1209 - accuracy: 0.0968
Epoch 17/100
484/484 [==============================] - 5s 11ms/step - loss: 5.0213 - accuracy: 0.1043
Epoch 18/100
484/484 [==============================] - 6s 12ms/step - loss: 4.9211 - accuracy: 0.1133
Epoch 19/100
484/484 [==============================] - 5s 11ms/step - loss: 4.8127 - accuracy: 0.1219
Epoch 20/100
484/484 [==============================] - 5s 11ms/step - loss: 4.7067 - accuracy: 0.1320
Epoch 21/100
484/484 [==============================] - 5s 10ms/step - loss: 4.6039 - accuracy: 0.1433
Epoch 22/100
484/484 [==============================] - 5s 11ms/step - loss: 4.5014 - accuracy: 0.1515
Epoch 23/100
484/484 [==============================] - 5s 10ms/step - loss: 4.3990 - accuracy: 0.1660
Epoch 24/100
484/484 [==============================] - 5s 11ms/step - loss: 4.2884 - accuracy: 0.1784
Epoch 25/100
484/484 [==============================] - 5s 11ms/step - loss: 4.1847 - accuracy: 0.1876
Epoch 26/100
484/484 [==============================] - 5s 11ms/step - loss: 4.0802 - accuracy: 0.2042
Epoch 27/100
484/484 [==============================] - 5s 11ms/step - loss: 3.9772 - accuracy: 0.2135
Epoch 28/100
484/484 [==============================] - 5s 11ms/step - loss: 3.8683 - accuracy: 0.2341
Epoch 29/100
484/484 [==============================] - 5s 11ms/step - loss: 3.7704 - accuracy: 0.2507
Epoch 30/100
484/484 [==============================] - 5s 11ms/step - loss: 3.6751 - accuracy: 0.2704
Epoch 31/100
484/484 [==============================] - 5s 11ms/step - loss: 3.5835 - accuracy: 0.2839
Epoch 32/100
484/484 [==============================] - 5s 10ms/step - loss: 3.4881 - accuracy: 0.3061
Epoch 33/100
484/484 [==============================] - 5s 10ms/step - loss: 3.4052 - accuracy: 0.3202
Epoch 34/100
484/484 [==============================] - 5s 10ms/step - loss: 3.3140 - accuracy: 0.3402
Epoch 35/100
484/484 [==============================] - 5s 10ms/step - loss: 3.2208 - accuracy: 0.3614
Epoch 36/100
484/484 [==============================] - 5s 10ms/step - loss: 3.1484 - accuracy: 0.3760
Epoch 37/100
484/484 [==============================] - 5s 11ms/step - loss: 3.0775 - accuracy: 0.3933
Epoch 38/100
484/484 [==============================] - 5s 11ms/step - loss: 3.0013 - accuracy: 0.4127
Epoch 39/100
484/484 [==============================] - 5s 10ms/step - loss: 2.9282 - accuracy: 0.4291
Epoch 40/100
484/484 [==============================] - 5s 10ms/step - loss: 2.8472 - accuracy: 0.4470
Epoch 41/100
484/484 [==============================] - 5s 11ms/step - loss: 2.7887 - accuracy: 0.4609
Epoch 42/100
484/484 [==============================] - 5s 11ms/step - loss: 2.7339 - accuracy: 0.4752
Epoch 43/100
484/484 [==============================] - 5s 10ms/step - loss: 2.6574 - accuracy: 0.4869
Epoch 44/100
484/484 [==============================] - 5s 11ms/step - loss: 2.6075 - accuracy: 0.4970
Epoch 45/100
484/484 [==============================] - 5s 11ms/step - loss: 2.5443 - accuracy: 0.5166
Epoch 46/100
484/484 [==============================] - 5s 11ms/step - loss: 2.4868 - accuracy: 0.5253
Epoch 47/100
484/484 [==============================] - 5s 11ms/step - loss: 2.4461 - accuracy: 0.5347
Epoch 48/100
484/484 [==============================] - 5s 11ms/step - loss: 2.3767 - accuracy: 0.5515
Epoch 49/100
484/484 [==============================] - 5s 11ms/step - loss: 2.3288 - accuracy: 0.5635
Epoch 50/100
484/484 [==============================] - 5s 10ms/step - loss: 2.2839 - accuracy: 0.5678
Epoch 51/100
484/484 [==============================] - 5s 10ms/step - loss: 2.2441 - accuracy: 0.5806
Epoch 52/100
484/484 [==============================] - 5s 10ms/step - loss: 2.1864 - accuracy: 0.5929
Epoch 53/100
484/484 [==============================] - 5s 11ms/step - loss: 2.1569 - accuracy: 0.5998
Epoch 54/100
484/484 [==============================] - 5s 11ms/step - loss: 2.1293 - accuracy: 0.6033
Epoch 55/100
484/484 [==============================] - 5s 10ms/step - loss: 2.0771 - accuracy: 0.6169
Epoch 56/100
484/484 [==============================] - 5s 10ms/step - loss: 2.0395 - accuracy: 0.6249
Epoch 57/100
484/484 [==============================] - 5s 10ms/step - loss: 1.9912 - accuracy: 0.6337
Epoch 58/100
484/484 [==============================] - 5s 11ms/step - loss: 1.9755 - accuracy: 0.6398
Epoch 59/100
484/484 [==============================] - 5s 10ms/step - loss: 1.9371 - accuracy: 0.6466
Epoch 60/100
484/484 [==============================] - 5s 11ms/step - loss: 1.9023 - accuracy: 0.6540
Epoch 61/100
484/484 [==============================] - 5s 11ms/step - loss: 1.8723 - accuracy: 0.6599
Epoch 62/100
484/484 [==============================] - 5s 11ms/step - loss: 1.8246 - accuracy: 0.6716
Epoch 63/100
484/484 [==============================] - 5s 10ms/step - loss: 1.8020 - accuracy: 0.6759
Epoch 64/100
484/484 [==============================] - 5s 10ms/step - loss: 1.7723 - accuracy: 0.6835
Epoch 65/100
484/484 [==============================] - 5s 10ms/step - loss: 1.7424 - accuracy: 0.6861
Epoch 66/100
484/484 [==============================] - 5s 10ms/step - loss: 1.7124 - accuracy: 0.6937
Epoch 67/100
484/484 [==============================] - 5s 10ms/step - loss: 1.6809 - accuracy: 0.7017
Epoch 68/100
484/484 [==============================] - 5s 11ms/step - loss: 1.6704 - accuracy: 0.7006
Epoch 69/100
484/484 [==============================] - 5s 11ms/step - loss: 1.6462 - accuracy: 0.7050
Epoch 70/100
484/484 [==============================] - 5s 11ms/step - loss: 1.6152 - accuracy: 0.7139
Epoch 71/100
484/484 [==============================] - 5s 11ms/step - loss: 1.5822 - accuracy: 0.7193
Epoch 72/100
484/484 [==============================] - 5s 11ms/step - loss: 1.5719 - accuracy: 0.7229
Epoch 73/100
484/484 [==============================] - 5s 10ms/step - loss: 1.5480 - accuracy: 0.7285
Epoch 74/100
484/484 [==============================] - 5s 11ms/step - loss: 1.5409 - accuracy: 0.7254
Epoch 75/100
484/484 [==============================] - 5s 11ms/step - loss: 1.5237 - accuracy: 0.7295
Epoch 76/100
484/484 [==============================] - 5s 11ms/step - loss: 1.4898 - accuracy: 0.7387
Epoch 77/100
484/484 [==============================] - 5s 11ms/step - loss: 1.4745 - accuracy: 0.7382
Epoch 78/100
484/484 [==============================] - 5s 11ms/step - loss: 1.4355 - accuracy: 0.7500
Epoch 79/100
484/484 [==============================] - 5s 11ms/step - loss: 1.4296 - accuracy: 0.7511
Epoch 80/100
484/484 [==============================] - 5s 10ms/step - loss: 1.4361 - accuracy: 0.7454
Epoch 81/100
484/484 [==============================] - 5s 11ms/step - loss: 1.4093 - accuracy: 0.7502
Epoch 82/100
484/484 [==============================] - 5s 10ms/step - loss: 1.3834 - accuracy: 0.7579
Epoch 83/100
484/484 [==============================] - 5s 10ms/step - loss: 1.3703 - accuracy: 0.7595
Epoch 84/100
484/484 [==============================] - 5s 10ms/step - loss: 1.3612 - accuracy: 0.7634
Epoch 85/100
484/484 [==============================] - 5s 11ms/step - loss: 1.3406 - accuracy: 0.7689
Epoch 86/100
484/484 [==============================] - 5s 10ms/step - loss: 1.3252 - accuracy: 0.7673
Epoch 87/100
484/484 [==============================] - 5s 10ms/step - loss: 1.3057 - accuracy: 0.7724
Epoch 88/100
484/484 [==============================] - 5s 10ms/step - loss: 1.2945 - accuracy: 0.7749
Epoch 89/100
484/484 [==============================] - 5s 10ms/step - loss: 1.2815 - accuracy: 0.7779
Epoch 90/100
484/484 [==============================] - 5s 11ms/step - loss: 1.2744 - accuracy: 0.7783
Epoch 91/100
484/484 [==============================] - 5s 10ms/step - loss: 1.2603 - accuracy: 0.7796
Epoch 92/100
484/484 [==============================] - 5s 10ms/step - loss: 1.2599 - accuracy: 0.7804
Epoch 93/100
484/484 [==============================] - 5s 10ms/step - loss: 1.2390 - accuracy: 0.7822
Epoch 94/100
484/484 [==============================] - 5s 11ms/step - loss: 1.2108 - accuracy: 0.7911
Epoch 95/100
484/484 [==============================] - 5s 11ms/step - loss: 1.2223 - accuracy: 0.7819
Epoch 96/100
484/484 [==============================] - 5s 10ms/step - loss: 1.2063 - accuracy: 0.7880
Epoch 97/100
484/484 [==============================] - 5s 10ms/step - loss: 1.1965 - accuracy: 0.7895
Epoch 98/100
484/484 [==============================] - 5s 10ms/step - loss: 1.1659 - accuracy: 0.7974
Epoch 99/100
484/484 [==============================] - 5s 11ms/step - loss: 1.1795 - accuracy: 0.7922
Epoch 100/100
484/484 [==============================] - 5s 10ms/step - loss: 1.1612 - accuracy: 0.7978