Amazing-Python-Scripts

1
# -*- coding: utf-8 -*-
2

3
import os
4
from tensorflow.keras.regularizers import l2
5
from tensorflow.keras.optimizers import Adam
6
from tensorflow.keras.layers import Dense
7
from tensorflow.keras.layers import Dropout
8
from tensorflow.keras.layers import Flatten
9
from tensorflow.keras.layers import Activation
10
from tensorflow.keras.layers import MaxPooling2D
11
from tensorflow.keras.layers import Conv2D
12
from tensorflow.keras.layers import BatchNormalization
13
from tensorflow.keras.models import Sequential
14
from tensorflow import keras
15
import tensorflow as tf
16
from imgaug import augmenters as iaa
17
import matplotlib.pyplot as plt
18
from skimage.color import rgb2gray
19
import numpy as np
20
import cv2 as cv
21
import csv
22
import random
23
PATH = '/content/Dataset'
24
TESTING_PATH = '/content/Dataset/Test'
25

26
# Minimal number of samples for each class for data augmentation
27
MIN_IMGS_IN_CLASS = 500
28

29
# Learning parameters
30
EPOCHS = 100
31
INIT_LR = 0.001
32
BATCH_SIZE = 256
33
SET_DECAY = True
34

35
# Imports
36

37
# images and corresponding labels
38
images = []
39
labels = []
40

41
# loop over all 43 classes
42
gtFile = open(PATH + '/Train.csv')
43
gtReader = csv.reader(gtFile, delimiter=',')  # csv parser for annotations file
44
next(gtReader)
45

46
# loop over all images in current annotations file
47
for row in gtReader:
48
    img = cv.imread(PATH + '/' + row[7])
49
    images.append(cv.resize(img, (28, 28)))
50
    labels.append(row[6])  # the 6th column is the label
51
gtFile.close()
52

53
print('Number of loaded images: ' + str(len(images)))
54
print('Number of loaded labels: ' + str(len(labels)))
55

56
train_X = np.asarray(images)
57
train_X = train_X / 255
58
train_X = np.asarray(train_X, dtype="float32")
59
train_Y = np.asarray(labels, dtype="float32")
60

61
print('Shape of training array: ' + str(train_X.shape))
62

63

64
def count_images_in_classes(lbls):
65
    """
66
    Determining number of images in a class for Graph
67
    :param lbls: Labels (different classes)
68
    :return: Dictionary of labels and count
69
    """
70
    dct = {}
71
    for i in lbls:
72
        if i in dct:
73
            dct[i] += 1
74
        else:
75
            dct[i] = 1
76
    return dct
77

78

79
samples_distribution = count_images_in_classes(train_Y)
80
print(samples_distribution)
81

82

83
def distribution_diagram(dct):
84
    """
85
    Histogram of Count and Label
86
    :param dct: Dictionary of labels and count
87
    :return: Histogram
88
    """
89
    plt.bar(range(len(dct)), list(dct.values()), align='center')
90
    plt.xticks(range(len(dct)), list(dct.keys()), rotation=90, fontsize=7)
91
    plt.show()
92

93

94
distribution_diagram(samples_distribution)
95

96

97
def preview(images, labels):
98
    """
99
    Preview of a Images rom a label
100
    :param images: Image from a Label
101
    :param labels: Label
102
    """
103
    plt.figure(figsize=(16, 16))
104
    for c in range(len(np.unique(labels))):
105
        i = random.choice(np.where(labels == c)[0])
106
        plt.subplot(10, 10, c + 1)
107
        plt.axis('off')
108
        plt.title('class: {}'.format(c))
109
        plt.imshow(images[i])
110

111

112
preview(train_X, train_Y)
113

114

115
def augment_imgs(imgs, p, imgaug=None):
116
    """
117
    Performs a set of augmentations with with a probability p
118
    :param imgs: Performs Data Augmentation
119
    :param p: Probability for augmentation
120
    """
121
    from imgaug import augmenters as iaa
122
    augs = iaa.SomeOf(
123
        (2, 4),
124
        [
125
            # crop images from each side by 0 to 4px (randomly chosen)
126
            iaa.Crop(px=(0, 4)),
127
            iaa.Affine(scale={
128
                "x": (0.8, 1.2),
129
                "y": (0.8, 1.2)
130
            }),
131
            iaa.Affine(translate_percent={
132
                "x": (-0.2, 0.2),
133
                "y": (-0.2, 0.2)
134
            }),
135
            # rotate by -45 to +45 degrees)
136
            iaa.Affine(rotate=(-45, 45)),
137
            # shear by -10 to +10 degrees
138
            iaa.Affine(shear=(-10, 10))
139
        ])
140

141
    seq = iaa.Sequential([iaa.Sometimes(p, augs)])
142
    res = seq.augment_images(imgs)
143
    return res
144

145

146
def augmentation(imgs, lbls):
147
    """
148
    Data Augmentation for a Label
149
    :param imgs: Images
150
    :param lbls: Labels
151
    """
152
    classes = count_images_in_classes(lbls)
153
    for i in range(len(classes)):
154
        if classes[i] < MIN_IMGS_IN_CLASS:
155
            # Number of samples to be added
156
            add_num = MIN_IMGS_IN_CLASS - classes[i]
157
            imgs_for_augm = []
158
            lbls_for_augm = []
159
            for j in range(add_num):
160
                im_index = random.choice(np.where(lbls == i)[0])
161
                imgs_for_augm.append(imgs[im_index])
162
                lbls_for_augm.append(lbls[im_index])
163
            augmented_class = augment_imgs(imgs_for_augm, 1)
164
            augmented_class_np = np.array(augmented_class)
165
            augmented_lbls_np = np.array(lbls_for_augm)
166
            imgs = np.concatenate((imgs, augmented_class_np), axis=0)
167
            lbls = np.concatenate((lbls, augmented_lbls_np), axis=0)
168
    return imgs, lbls
169

170

171
train_X, train_Y = augmentation(train_X, train_Y)
172

173
print(train_X.shape)
174
print(train_Y.shape)
175

176
augmented_samples_distribution = count_images_in_classes(train_Y)
177
print(augmented_samples_distribution)
178

179
distribution_diagram(augmented_samples_distribution)
180

181
preview(train_X, train_Y)
182

183
train_X = rgb2gray(train_X)
184

185

186
def build(width, height, depth, classes):
187
    """
188
    Initialize the model and its dimension
189
    """
190
    model = keras.Sequential()
191
    inputShape = (height, width, depth)
192
    chanDim = -1
193

194
    # CONV => RELU => BN => POOL
195
    model.add(Conv2D(8, (5, 5), padding="same", input_shape=inputShape))
196
    model.add(Activation("relu"))
197
    model.add(BatchNormalization(axis=chanDim))
198
    model.add(MaxPooling2D(pool_size=(2, 2)))
199

200
    # first set of (CONV => RELU => CONV => RELU) * 2 => POOL
201
    model.add(Conv2D(16, (3, 3), padding="same"))
202
    model.add(Activation("relu"))
203
    model.add(BatchNormalization(axis=chanDim))
204
    model.add(Conv2D(16, (3, 3), padding="same"))
205
    model.add(Activation("relu"))
206
    model.add(BatchNormalization(axis=chanDim))
207
    model.add(MaxPooling2D(pool_size=(2, 2)))
208

209
    # second set of (CONV => RELU => CONV => RELU) * 2 => POOL
210
    model.add(Conv2D(32, (3, 3), padding="same"))
211
    model.add(Activation("relu"))
212
    model.add(BatchNormalization(axis=chanDim))
213
    model.add(Conv2D(32, (3, 3), padding="same"))
214
    model.add(Activation("relu"))
215
    model.add(BatchNormalization(axis=chanDim))
216
    model.add(MaxPooling2D(pool_size=(2, 2)))
217

218
    # first set of FC => RELU layers
219
    model.add(Flatten())
220
    model.add(Dense(128))
221
    model.add(Activation("relu"))
222
    model.add(BatchNormalization())
223
    model.add(Dropout(0.5))
224

225
    # second set of FC => RELU layers
226
    model.add(Flatten())
227
    model.add(Dense(128))
228
    model.add(Activation("relu"))
229
    model.add(BatchNormalization())
230
    model.add(Dropout(0.5))
231

232
    # softmax classifier
233
    model.add(Dense(classes))
234
    model.add(Activation("softmax"))
235

236
    # return the constructed network architecture
237
    return model
238

239

240
model = build(28, 28, 1, 43)
241

242
if SET_DECAY:
243
    opt = Adam(lr=INIT_LR, decay=INIT_LR / (EPOCHS * 0.5))
244
else:
245
    opt = Adam(lr=INIT_LR)
246
model.compile(loss="sparse_categorical_crossentropy",
247
              optimizer=opt,
248
              metrics=["accuracy"])
249

250
test_images = []
251
test_labels = []
252
test = '/content/Dataset'
253
# loop over all 43 classes
254
gtFile = open('/content/Dataset/Test.csv')  # annotations file
255
gtReader = csv.reader(gtFile, delimiter=',')  # csv parser for annotations file
256
next(gtReader)
257

258
# loop over all images in current annotations file
259
for row in gtReader:
260
    img = cv.imread(PATH + '/' + row[7])
261
    test_images.append(cv.resize(img, (28, 28)))
262
    test_labels.append(row[6])
263
gtFile.close()
264

265
test_X = np.asarray(test_images)
266
test_X = test_X / 255
267
test_X = np.asarray(test_X, dtype="float32")
268

269
test_X = rgb2gray(test_X)
270

271
test_Y = np.asarray(test_labels, dtype="float32")
272

273
print(train_X.shape)
274
train_X_ext = np.expand_dims(train_X, axis=3)
275
print(train_X_ext.shape)
276
print(train_Y.shape)
277
train_Y_ext = np.expand_dims(train_Y, axis=1)
278
print(train_Y_ext.shape)
279

280
with tf.device('/device:GPU:0'):
281
    os.mkdir("models_Xception")
282
    for i in range(EPOCHS):
283
        model.save("models_Xception/model_" + str(i) + ".h5")
284
        H = model.fit(train_X_ext, train_Y, epochs=1, batch_size=BATCH_SIZE)
285

286
print(model.summary())
287

288
model.save_weights('model_final.h5')
289