Deep Learning to Recognize Traffic Signs with Python GUI: Part 1

This content is powered by Balige Publishing. Visit this link (collaboration with Rismon Hasiholan Sianipar)

Step 1: Now, you will create a GUI to implement how to recognizing traffic signs. Open Qt Designer and choose Main Window template. Save the form as gui_traffic.ui.

Step 2: Put one Widget from Containers panel onto form and set its ObjectName property as widgetGraph. You will use this widget to plot graph.

Step 3: Put two Push Button widgets onto form. Set their text properties as LOAD DATA and TRAINING MODEL. Set their objectName properties as pbLoad and pbTraining.

Step 4: Put a Group Box widget onto form and set its objectName property as gbHistogram. Inside it, put two Radio Button widgets and set their objectName properties as rbHistTraining and rbHistTesting. Set their text properties as Training Data and Testing Data.

Step 5: Put another Group Box widget onto form and set its objectName property as gbCorr. Inside it, put two Radio Button widgets and set their objectName properties as rbCorrTraining and rbCorrTesting. Set their text properties as Training Data and Testing Data.

Step 6: Put one Combo Box widget onto form and set its objectName property as cbAccuracy. Double click on this widget and populate it with two items as shown in Figure below.

Step 7: Put the third Group Box widget onto form and set its objectName property as gbDataset. Inside it, put two Radio Button widgets and set their objectName properties as rbDataTraining and rbDataTesting. Set their text properties as Training Data and Testing Data.

Step 8: Put one Table Widget onto the right side of form and set its objectName property as twData. Save the form. It now looks as shown in Figure below.

Step 9: Right click on widgetGraph and choose Promote to …. Set Promoted class name as widgetClass. Click Add and Promote button. In Object Inspector window, you can see that widgetGraph is now an object of widget_class as shown in Figure below.

Step 10: Define the definition of widget_class and save it as widget_class.py as follows:

#widget_class.py

from PyQt5.QtWidgets import*
from matplotlib.backends.backend_qt5agg import FigureCanvas
from matplotlib.figure import Figure
    
class widget_class(QWidget):    
    def __init__(self, parent = None):
        QWidget.__init__(self, parent)        
        self.canvas = FigureCanvas(Figure())
       
        vertical_layout = QVBoxLayout()
        vertical_layout.addWidget(self.canvas)
        
        self.canvas.axis1 = self.canvas.figure.add_subplot(111)
        self.canvas.figure.set_facecolor("xkcd:neon yellow")
        self.setLayout(vertical_layout)

Step 11: Write this Python script and save it as recognize_traffic_sign.py:

#recognize_traffic_sign.py
from PyQt5.QtWidgets import *
from PyQt5.uic import loadUi
from matplotlib.backends.backend_qt5agg import (NavigationToolbar2QT as NavigationToolbar)
from matplotlib.colors import ListedColormap

class DemoGUI_TrafficSign(QMainWindow):   
    def __init__(self):       
        QMainWindow.__init__(self)
        loadUi("gui_traffic.ui",self)

        self.setWindowTitle("GUI Demo of Recognizing Traffic Signs")
        self.addToolBar(NavigationToolbar(self.widgetGraph.canvas,\
            self))
                      
if __name__ == '__main__':
    import sys
    app = QApplication(sys.argv)
    ex = DemoGUI_TrafficSign()
    ex.show()
    sys.exit(app.exec_())

Step 12: Run recognize_traffic_sign.py and see the form as shown in Figure below.

Step 13: Define set_state() method to determine the state of some widgets. This method is used to disble some widgets when the form first runs:

def set_state(self,state):
    self.gbHistogram.setEnabled(state)
    self.gbCorr.setEnabled(state)
    self.pbTraining.setEnabled(state)
    self.cbAccuracy.setEnabled(state)
    self.gbDataset.setEnabled(state)

Step 14: Call set_state() method and put it inside __init__() method:

def __init__(self):       
    QMainWindow.__init__(self)
    loadUi("gui_traffic.ui",self)
    self.setWindowTitle("GUI Demo of Recognizing Traffic Signs")
    self.addToolBar(NavigationToolbar(self.widgetGraph.canvas, self))
    self.set_state(False)

Step 15: Import all needed libraries into recognize_traffic_sign.py using the following statements:

import numpy as np 
import pandas as pd 
import matplotlib.pyplot as plt
import seaborn as sns
import cv2
from PIL import Image
import os
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential,load_model
from tensorflow.keras.layers import Dense, Dropout, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Dropout
from tensorflow.keras import backend as K
from sklearn.metrics import accuracy_score

Step 16: Define load_data() and create_dataset_dataframe() methods to load training and testing data and to create dataframes:

def load_data(self):
    self.data = []
    self.labels = []
    classes = 43
    self.curr_path = os.getcwd()

    #Retrieving the images and their labels 
    for i in range(classes):
        path = os.path.join(self.curr_path,'train',str(i))
        images = os.listdir(path)
        for a in images:
            try:
                image = Image.open(path + '\\'+ a)
                image = image.resize((30,30))
                image = np.array(image)
                self.data.append(image)
                self.labels.append(i)
            except:
                print("Error loading image")

    #Converting lists into numpy arrays
    self.data = np.array(self.data)
    self.labels = np.array(self.labels)
    print(self.data.shape, self.labels.shape)

    table = {'image_path': path, 'target': self.labels}
    self.df = pd.DataFrame(data=table)
    self.df = self.df.sample(frac = 1).reset_index(drop=True) 
        
    #Creates dataset and dataframe
    self.create_dataset_dataframe()
        
    #Disables pbLoad widget
    self.pbLoad.setEnabled(False)
        
    #Enables back widgets
    self.set_state(True)
        
    #Checks rbDataTraining widget
    self.rbDataTraining.setChecked(True)

def create_dataset_dataframe(self):
    #Splitting training and testing dataset
    self.X_train, self.X_test, self.y_train, self.y_test = \
        train_test_split(self.data, self.labels, test_size=0.2, \
        random_state=42)
    print(self.X_train.shape, self.X_test.shape, \
        self.y_train.shape, self.y_test.shape)

    #Converting the labels into one hot encoding
    self.y_train = to_categorical(self.y_train, 43)
    self.y_test = to_categorical(self.y_test, 43)
        
    #Creates testing dataframe
    self.test_df = pd.read_csv(str(self.curr_path) +'/Test.csv')

    #Creates training dataframe
    self.df_train = pd.read_csv(str(self.curr_path) +'/Train.csv')
    nRow, nCol = self.df_train.shape
    print(f'There are {nRow} rows and {nCol} columns')
    print(self.df_train.head(5))

Step 17: Define display_table() method to display testing and training data on table widget as follows:

def display_table(self,df, tableWidget):
    # show data on table widget
    self.write_df_to_qtable(df,tableWidget)
        
    styleH = "::section {""background-color: red; }"
    tableWidget.horizontalHeader().setStyleSheet(styleH)

    styleV = "::section {""background-color: red; }"
    tableWidget.verticalHeader().setStyleSheet(styleV)  

# Takes a df and writes it to a qtable provided. df headers 
# become qtable headers
@staticmethod
def write_df_to_qtable(df,table):
    headers = list(df)
    table.setRowCount(df.shape[0])
    table.setColumnCount(df.shape[1]) 
    table.setHorizontalHeaderLabels(headers)

    # getting data from df is computationally costly 
    # so convert it to array first
    df_array = df.values
    for row in range(df.shape[0]):
        for col in range(df.shape[1]):
            table.setItem(row, col, \                                
                QTableWidgetItem(str(df_array[row,col])))

Step 18: Invoke display_table() method and put it at the end of load_data():

#Displays data on table widget
self.display_table(self.df_train,self.twData)

Step 19: Run recognize_traffic_sign.py to see that training data has been displayed on table widget as shown in Figure below.

Step 20: Define rb_dataset() method to read text property of rbDataTraining and rbDataTesting widgets to determine which data will be displayed on twData widget:

def rb_dataset(self,b):	
    if b.text() == "Training Data":
        if b.isChecked() == True:
            self.display_table(self.df_train,self.twData)
        else:
            self.display_table(self.test_df,self.twData)
                
    if b.text() == "Testing Data":
        if b.isChecked() == True:
            self.display_table(self.test_df,self.twData)
        else:
            self.display_table(self.df_train,self.twData)

Step 21: Connect toggled() event of both rbDataTraining and rbDataTesting widgets with rb_dataset method and put them inside __init__() method as follows:

def __init__(self):       
    QMainWindow.__init__(self)
    loadUi("gui_traffic.ui",self)
    self.setWindowTitle("GUI Demo of Recognizing Traffic Signs")
    self.addToolBar(NavigationToolbar(self.widgetGraph.canvas, self))
    self.set_state(False)
    self.pbLoad.clicked.connect(self.load_data)
    self.rbDataTraining.toggled.connect(\
        lambda:self.rb_dataset(self.rbDataTraining))
    self.rbDataTesting.toggled.connect(\
        lambda:self.rb_dataset(self.rbDataTesting))

Step 22: Run recognize_traffic_sign.py to see that training data has been displayed on table widget. Then click on Testing Data radio button on gbDataset group box to see testing data displayed on table widget as shown in Figure below.

Step 23: Define display_histogram() method to display distribution of number of images in each class:

def display_histogram(self, widget, df, xlabel, ylabel, title):   
    widget.canvas.axis1.clear()
    x=df.target.value_counts()
    sns.barplot(x.index,x, ax=widget.canvas.axis1)
    widget.canvas.axis1.set_xlabel(xlabel)
    widget.canvas.axis1.set_ylabel(ylabel)
    widget.canvas.axis1.set_title(title)
    widget.canvas.axis1.grid()
    widget.canvas.draw()

Step 24: Invoke display_histogram() method and make rbHistTraining widget checked when form first runs. Put them at the end of load_data() method:

#Displays histogram of training data
hist_train = self.df.target.value_counts()
self.display_histogram(self.widgetGraph, hist_train, \
    'Class', 'Samples', \
    'The distribution of number of training samples in each class')
#Checks rbDataTraining widget
self.rbHistTraining.setChecked(True)

Step 25: Run recognize_traffic_sign.py to see distribution of number of images in each class of training data as shown in Figure below.

Step 26: Define rb_histogram() method to read text property of rbHistTraining and rbHistTesting widgets to determine which data will be displayed on widgetGraph widget:

def rb_histogram(self,b):
    hist_train = self.df.target.value_counts()
    hist_test = self.test_df.ClassId.value_counts()
    if b.text() == "Training Data":
        if b.isChecked() == True:
            self.display_histogram(self.widgetGraph, hist_train, \
                'Class', 'Samples', \
                'The distribution of number of training samples in 
                each class')
        else:
            self.display_histogram(self.widgetGraph, hist_test, \
                'Class', 'Samples', \
                'The distribution of number of testing samples 
                in each class')
                
    if b.text() == "Testing Data":
        if b.isChecked() == True:
            self.display_histogram(self.widgetGraph, hist_test, \
                'Class', 'Samples', \
                'The distribution of number of testing samples 
                in each class')
        else:
            self.display_histogram(self.widgetGraph, hist_train, \
                'Class', 'Samples', 'The distribution of number of\
                training samples in each class')

Step 27: Connect toggled() event of both rbHistTraining and rbHistTesting widgets with rb_histogram method and put them inside __init__() method as follows:

def __init__(self):       
    QMainWindow.__init__(self)
    loadUi("gui_traffic.ui",self)
    self.setWindowTitle("GUI Demo of Recognizing Traffic Signs")
    self.addToolBar(NavigationToolbar(self.widgetGraph.canvas, self))
    self.set_state(False)
    self.pbLoad.clicked.connect(self.load_data)
    self.rbDataTraining.toggled.connect(\
        lambda:self.rb_dataset(self.rbDataTraining))
    self.rbDataTesting.toggled.connect(\
        lambda:self.rb_dataset(self.rbDataTesting))       
    self.rbHistTraining.toggled.connect(\
        lambda:self.rb_histogram(self.rbHistTraining))
    self.rbHistTesting.toggled.connect(\
        lambda:self.rb_histogram(self.rbHistTesting))

Step 28: Run recognize_traffic_sign.py to see distribution of number of images in each class of training data. Then, click on Testing Data in gbHistogram group box. You will see the histogram of testing data as shown in Figure below.

Deep Learning to Recognize Traffic Signs with Python GUI: Part 2

Deep Learning to Predict Handwritten Digits with Python GUI: Part 2

This content is powered by Balige Publishing. Visit this link (collaboration with Rismon Hasiholan Sianipar) PART 1

You will improve this model's performance by preprocessing the input image.

Step 1: Import these statements into identify_digit.py:

import cv2
import imutils
import scipy
from imutils.perspective import four_point_transform
from scipy import ndimage

Step 2: Define imageprepare() method to invert image to get whie digit on black background, convert image to grayscale, resize image to be 28x28, fit the image into this 20x20 pixel box, and resize outer box to fit it into a 20x20 box:

def imageprepare(self,fileName):       
    image = cv2.imread(fileName)

    #Invert image to get whie digit on black background
    image = cv2.bitwise_not(image)

    #Converts image to grayscale
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    #resizes image to be 28x28
    (thresh, gray) = cv2.threshold(gray, 128, 255, \
        cv2.THRESH_BINARY | cv2.THRESH_OTSU)
    
    #First you need to fit the images into this 20x20 pixel box. 
    #Therefore you need to remove every row and column at the sides 
    #of the image which are completely black.
    while np.sum(gray[0]) == 0:
        gray = gray[1:]

    while np.sum(gray[:, 0]) == 0:
        gray = np.delete(gray, 0, 1)

    while np.sum(gray[-1]) == 0:
        gray = gray[:-1]

    while np.sum(gray[:, -1]) == 0:
        gray = np.delete(gray, -1, 1)

    rows, cols = gray.shape

    #Now you can resize our outer box to fit it into a 20x20 box. 
    #You need a resize factor for this.
    if rows > cols:
        factor = 20.0 / rows
        rows = 20
        cols = int(round(cols * factor))
        gray = cv2.resize(gray, (cols, rows))

    else:
        factor = 20.0 / cols
        cols = 20
        rows = int(round(rows * factor))
        gray = cv2.resize(gray, (cols, rows))

    #At the end ye need a 28x28 pixel image so we add the missing 
    #black rows and columns 
    #using the np.lib.pad function which adds 0s to the sides.
    colsPadding = (int(np.math.ceil((28 - cols) / 2.0)), \
        int(np.math.ceil((28 - cols) / 2.0)))
    rowsPadding = (int(np.math.ceil((28 - rows) / 2.0)), \
        int(np.math.ceil((28 - rows) / 2.0)))
    gray = np.lib.pad(gray, (rowsPadding, colsPadding), 'constant')
    #gray = np.lib.pad(gray, 2, 'constant',constant_values=255)
        
    shiftx, shifty = self.getBestShift(gray)
    shifted = self.shift(gray, shiftx, shifty)
    gray = shifted 
    gray = cv2.resize(gray, (28, 28))
    cv2.imwrite("result.png", gray)
    return gray       
        
def getBestShift(self,img):
    cy, cx = ndimage.measurements.center_of_mass(img)

    rows, cols = img.shape
    shiftx = np.round(cols / 2.0 - cx).astype(int)
    shifty = np.round(rows / 2.0 - cy).astype(int)

    return shiftx, shifty

def shift(self,img, sx, sy):
    rows, cols = img.shape
    M = np.float32([[1, 0, sx], [0, 1, sy]])
    shifted = cv2.warpAffine(img, M, (cols, rows))
    return shifted

Step 3: Replace predict_digit() function with this code:

def predict_digit(self, img):  
    model = load_model('mnist.h5')
         
    #Normalize image to range [0 1]
    img = cv2.normalize(img, None, alpha=0, beta=1, \
        norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)

    #Reshapes image
    img = img.reshape(1,28,28,1)       
        
    #predicting the class
    res = model.predict([img])[0]
    return np.argmax(res), max(res)

Step 4: Run identify_digit.py, draw a digit, then click PREDICT button, and you will get the predicted digit and its accuracy as shown in Figure below.

Step 5: Click CLEAR button and redraw a digit. Then, click PREDICT button, and you will get the predicted digit and its accuracy as shown in Figure below.

Below is the full script of  identify_digit.py so far:

#identify_digit.py
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.uic import loadUi
from matplotlib.backends.backend_qt5agg import (NavigationToolbar2QT as NavigationToolbar)
from widget_paint import widget_paint
from PIL import ImageGrab, Image
import numpy as np
from tensorflow.keras.models import load_model

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.datasets import mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from tensorflow.keras import backend as K

import cv2
import imutils
import scipy
from imutils.perspective import four_point_transform
from scipy import ndimage

class Identify_Digit(QMainWindow):   
    def __init__(self):       
        QMainWindow.__init__(self)
        loadUi("predict_mnist.ui",self)
        self.setWindowTitle("Predicting Digits with CNN")
        self.pbTrain.clicked.connect(self.load_train)
        self.pbClear.clicked.connect(self.clear_canvas)
        self.pbSave.clicked.connect(self.save_canvas)
        self.pbPredict.clicked.connect(self.classify_handwriting)

    def load_train(self):
        #Loads data and trains model
        # the data, split between train and test sets
        (x_train, y_train), (x_test, y_test) = mnist.load_data()
        print(x_train.shape, y_train.shape)

        #Preprocesses the data
        x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
        x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
        input_shape = (28, 28, 1)

        # convert class vectors to binary class matrices
        y_train = keras.utils.to_categorical(y_train, num_classes=None)
        y_test = keras.utils.to_categorical(y_test, num_classes=None)
        x_train = x_train.astype('float32')
        x_test = x_test.astype('float32')
        x_train /= 255
        x_test /= 255
        print('x_train shape:', x_train.shape)
        print(x_train.shape[0], 'train samples')
        print(x_test.shape[0], 'test samples')

        #Creates the model
        batch_size = 128
        num_classes = 10
        epochs = 10

        model = Sequential()
        model.add(Conv2D(32, kernel_size=(3, 3),activation='relu',input_shape=input_shape))
        model.add(Conv2D(64, (3, 3), activation='relu'))
        model.add(MaxPooling2D(pool_size=(2, 2)))
        model.add(Dropout(0.25))
        model.add(Flatten())
        model.add(Dense(256, activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(num_classes, activation='softmax'))

        model.compile(loss=keras.losses.categorical_crossentropy,optimizer=keras.optimizers.Adadelta(),metrics=['accuracy'])

        #Trains the model
        hist = model.fit(x_train, y_train,batch_size=batch_size,epochs=epochs,verbose=1,validation_data=(x_test, y_test))
        print("The model has successfully trained")

        model.save('mnist.h5')
        print("Saving the model as mnist.h5")

        #Evaluates the model
        score = model.evaluate(x_test, y_test, verbose=0)
        print('Test loss:', score[0])
        print('Test accuracy:', score[1])
        
        pbTrain.setEnabled(False)
        
	# method for clearing every thing on canvas
    def clear_canvas(self):
        self.widgetDigit = widget_paint(self)
        self.widgetDigit.clear()
        self.widgetDigit.show()

	# method for saving image on canvas
    def save_canvas(self):
        self.widgetDigit.save()

    def predict_digit(self, img):  
        model = load_model('mnist.h5')
         
        #Normalize image to range [0 1]
        img = cv2.normalize(img, None, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)

        #Reshapes image
        img = img.reshape(1,28,28,1)       
        
        #predicting the class
        res = model.predict([img])[0]
        return np.argmax(res), max(res)

    def classify_handwriting(self):
        self.widgetDigit.return_image()        
        #im = Image.open("test.png")
        
        im = self.imageprepare("test.png")
        digit, acc = self.predict_digit(im)
        self.labelPredict1.setText('Predicted= '+str(digit))
        self.labelPredict2.setText('Accuracy= '+ str(int(acc*100)))

    def imageprepare(self,fileName):       
        image = cv2.imread(fileName)

        #Invert image to get whie digit on black background
        image = cv2.bitwise_not(image)

        #Converts image to grayscale
        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

        #resizes image to be 28x28
        (thresh, gray) = cv2.threshold(gray, 128, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
    
        #First we want to fit the images into this 20x20 pixel box. 
        #Therefore we need to remove every row and column at the sides of the image 
        #which are completely black.
        while np.sum(gray[0]) == 0:
            gray = gray[1:]

        while np.sum(gray[:, 0]) == 0:
            gray = np.delete(gray, 0, 1)

        while np.sum(gray[-1]) == 0:
            gray = gray[:-1]

        while np.sum(gray[:, -1]) == 0:
            gray = np.delete(gray, -1, 1)

        rows, cols = gray.shape

        #Now we want to resize our outer box to fit it into a 20x20 box. 
        #We need a resize factor for this.
        if rows > cols:
            factor = 20.0 / rows
            rows = 20
            cols = int(round(cols * factor))
            gray = cv2.resize(gray, (cols, rows))

        else:
            factor = 20.0 / cols
            cols = 20
            rows = int(round(rows * factor))
            gray = cv2.resize(gray, (cols, rows))

        #at the end we need a 28x28 pixel image so we add the missing black rows and columns 
        #using the np.lib.pad function which adds 0s to the sides.
        colsPadding = (int(np.math.ceil((28 - cols) / 2.0)), int(np.math.ceil((28 - cols) / 2.0)))
        rowsPadding = (int(np.math.ceil((28 - rows) / 2.0)), int(np.math.ceil((28 - rows) / 2.0)))
        gray = np.lib.pad(gray, (rowsPadding, colsPadding), 'constant')
        #gray = np.lib.pad(gray, 2, 'constant',constant_values=255)
        
        shiftx, shifty = self.getBestShift(gray)
        shifted = self.shift(gray, shiftx, shifty)
        gray = shifted 
        gray = cv2.resize(gray, (28, 28))
        cv2.imwrite("result.png", gray)
        return gray       
        
    def getBestShift(self,img):
        cy, cx = ndimage.measurements.center_of_mass(img)

        rows, cols = img.shape
        shiftx = np.round(cols / 2.0 - cx).astype(int)
        shifty = np.round(rows / 2.0 - cy).astype(int)

        return shiftx, shifty

    def shift(self,img, sx, sy):
        rows, cols = img.shape
        M = np.float32([[1, 0, sx], [0, 1, sy]])
        shifted = cv2.warpAffine(img, M, (cols, rows))
        return shifted
        
if __name__ == '__main__':
    import sys
    app = QApplication(sys.argv)
    ex = Identify_Digit()
    ex.show()
    sys.exit(app.exec_())