Deep Learning to Predict Handwritten Digits with Python GUI: Part 1

This content is powered by Balige Publishing. Visit this link (collaboration with Rismon Hasiholan Sianipar)

Step 1: Open Qt Designer. Create a form with Main Window template. Put a Widget from Containers panel onto form. Set its objectName property as widgetDigit. Set its both Width and Height properties to 400.

Step 2: In the right side of widgetDigit, put a Push Button widget and set its objectName property as pbTrain. Set its text property as TRAIN MODEL.

Step 3: Below the widgetDigit, put two Push Button widgets and set their objectName properties as pbClear and pbSave. Set their text properties as CLEAR and SAVE.

Step 4: In the right side of widgetDigit, put two Label widgets and set their objectName as labelPredict1 and labelPredict2.

Step 5: Below them, put another Push Button widget and set its objectName property as pbPredict. Set its text property as pbPredict. Set its text property as PREDICT.

Step 6: Then, right click on widgetDigit and choose Promote to … menu item. Set promoted class name as widget_paint. In Object Inspector window, you can see that widgetDigit now is of widget_paint class.

Step 7: Save the form as predict_mnist.ui as shown in Figure below.

Step 8: Now, you write the definition of widget_paint class and save it as widget_paint.py as follows:

#widget_paint.py
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
import cv2   
import numpy as np
    
class widget_paint(QWidget):    
    def __init__(self, parent = None):
        QWidget.__init__(self, parent) 
        
        # setting geometry
        self.setGeometry(20, 20, 400, 400)
        
        # creating image object
        self.image = QImage(self.size(), QImage.Format_RGB32)
        
        # making image color to white
        self.image.fill(Qt.white)
        # variables
        # drawing flag
        self.drawing = False
        
        # default brush size
        self.brushSize = 40
        
        # default color
        self.brushColor = Qt.black
        
        # QPoint object to trace the point
        self.lastPoint = QPoint()
        
    # method for checking mouse cicks
    def mousePressEvent(self, event):
        # if left mouse button is pressed
        if event.button() == Qt.LeftButton:
            # make drawing flag true
            self.drawing = True
            # make last point to the point of cursor
            self.lastPoint = event.pos()

    # method for tracking mouse activity
    def mouseMoveEvent(self, event):		
        # checking if left button is pressed and drawing flag is true
        if (event.buttons() & Qt.LeftButton) & self.drawing:
            # creating painter object
            painter = QPainter(self.image)
			
            # set the pen of the painter
            painter.setPen(QPen(self.brushColor, self.brushSize,\
 		Qt.SolidLine, Qt.RoundCap, Qt.RoundJoin))
			
            # draw line from the last point of cursor to the current point
	   # this will draw only one step
            painter.drawLine(self.lastPoint, event.pos())
			
	   # change the last point
            self.lastPoint = event.pos()
            # update
            self.update()    
    
    # method for mouse left button release
    def mouseReleaseEvent(self, event):
        if event.button() == Qt.LeftButton:
            # make drawing flag false
            self.drawing = False

    # paint event
    def paintEvent(self, event):
        # create a canvas
        canvasPainter = QPainter(self)
		
        # draw rectangle on the canvas
        canvasPainter.drawImage(self.rect(), self.image, \
            self.image.rect())

    def return_image(self):
        cv2.imwrite('test.png', self.QImageToCvMat(self.image))
    
    # method for clearing every thing on canvas
    def clear(self):
        # make the whole canvas white
        self.image.fill(Qt.white)
        # update
        self.update()
    
    # method for saving canvas
    def save(self):
        filePath, _ = QFileDialog.getSaveFileName(self, \
            "Save Image", "",\
            "PNG(*.png);;JPEG(*.jpg *.jpeg);;All Files(*.*) ")
        if filePath == "":
            return
        self.image.save(filePath)

    def QImageToCvMat(self,incomingImage):
        '''  Converts a QImage into an opencv MAT format  '''

        incomingImage = \
           incomingImage.convertToFormat(QImage.Format.Format_RGBA8888)

        width = incomingImage.width()
        height = incomingImage.height()

        ptr = incomingImage.bits()
        ptr.setsize(height * width * 4)
        arr = np.frombuffer(ptr, np.uint8).reshape((height, width, 4))
        return arr

Step 9: Create a new Python script and save it as identify_digit.py as follows:

#identify_digit.py
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.uic import loadUi
from matplotlib.backends.backend_qt5agg import (NavigationToolbar2QT as NavigationToolbar)
from widget_paint import widget_paint
from PIL import ImageGrab, Image
import numpy as np
from tensorflow.keras.models import load_model

class Identify_Digit(QMainWindow):   
    def __init__(self):       
        QMainWindow.__init__(self)
        loadUi("predict_mnist.ui",self)
        self.setWindowTitle("Predicting Digits with CNN")

if __name__ == '__main__':
    import sys
    app = QApplication(sys.argv)
    ex = Identify_Digit()
    ex.show()
    sys.exit(app.exec_())

Step 10: Run identify_digit.py and you will get the result as shown in Figure below.

Step 11: Import all tensorflow libraries needed as follows:

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.datasets import mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from tensorflow.keras import backend as K

Step 12: Define load_train() method to load MNIST dataset and train CNN model:

def load_train(self):
    #Loads data and trains model
    # the data, split between train and test sets
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    print(x_train.shape, y_train.shape)

    #Preprocesses the data
    x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
    x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
    input_shape = (28, 28, 1)

    # convert class vectors to binary class matrices
    y_train = keras.utils.to_categorical(y_train, num_classes=None)
    y_test = keras.utils.to_categorical(y_test, num_classes=None)
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    x_train /= 255
    x_test /= 255
    print('x_train shape:', x_train.shape)
    print(x_train.shape[0], 'train samples')
    print(x_test.shape[0], 'test samples')

    #Creates the model
    batch_size = 128
    num_classes = 10
    epochs = 10

    model = Sequential()
    model.add(Conv2D(32, kernel_size=(3, 3),\
        activation='relu',input_shape=input_shape))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(256, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='softmax'))

    model.compile(loss=keras.losses.categorical_crossentropy,\
        optimizer=keras.optimizers.Adadelta(),metrics=['accuracy'])

    #Trains the model
    hist = model.fit(x_train, y_train,\
        batch_size=batch_size,epochs=epochs,verbose=1,\
        validation_data=(x_test, y_test))
    print("The model has successfully trained")

    model.save('mnist.h5')
    print("Saving the model as mnist.h5")

    #Evaluates the model
    score = model.evaluate(x_test, y_test, verbose=0)
    print('Test loss:', score[0])
    print('Test accuracy:', score[1])
        
    pbTrain.setEnabled(False)

Step 13: Connect clicked() event of pbTrain with load_train() method and put it inside __init__() method as follows:

def __init__(self):       
    QMainWindow.__init__(self)
    loadUi("predict_mnist.ui",self)
    self.setWindowTitle("Predicting Digits with CNN")
    self.pbTrain.clicked.connect(self.load_train)

Step 14: Run identify_digit.py. Click on TRAIN MODEL button to load and create CNN model. On widgetDigit (one with white background), you can draw something like digit as shown in Figure below.

Step 15: In identify_digit.py, define clear_canvas() method to clear everything in widgetDigit as follows:

# method for clearing every thing on canvas
def clear_canvas(self):
    self.widgetDigit = widget_paint(self)
    self.widgetDigit.clear()
    self.widgetDigit.show()

Step 16: Then, define save_canvas() method to save the content of widgetDigit as an image:

# method for saving image on canvas
def save_canvas(self):
    self.widgetDigit.save()

Step 17: Connect clicked() event of pbSave widget with clear_canvas() function and that of pbSave widget with save_canvas() function. Put them inside __init__() method:

def __init__(self):       
    QMainWindow.__init__(self)
    loadUi("predict_mnist.ui",self)
    self.setWindowTitle("Predicting Digits with CNN")
    self.pbTrain.clicked.connect(self.load_train)
    self.pbClear.clicked.connect(self.clear_canvas)
    self.pbSave.clicked.connect(self.save_canvas)

Step 18: Define predict_digit() function to resize the image, convert it to grayscale, reshape it, and predict the class:

def predict_digit(self, img):
    model = load_model('mnist.h5')

    #resize image to 28x28 pixels
    img = img.resize((28,28))
    
    #convert rgb to grayscale
    img = img.convert('L')
    img = np.array(img)
    
    #reshaping to support our model input and normalizing
    img = img.reshape(1,28,28,1)
    img = img/255.0        
        
    #predicting the class
    res = model.predict([img])[0]
    return np.argmax(res), max(res)

Step 19: Define classify_handwriting() method to classify the digit:

def classify_handwriting(self):
    self.widgetDigit.return_image()        
    im = Image.open("test.png")
        
    digit, acc = self.predict_digit(im)
    self.labelPredict1.setText('Predicted= '+str(digit))
    self.labelPredict2.setText('Accuracy= '+ str(int(acc*100)))

Step 20: Connect clicked() event of pbPredict widget with classify_handwriting() function and put it inside __init__() method:

def __init__(self):       
    QMainWindow.__init__(self)
    loadUi("predict_mnist.ui",self)
    self.setWindowTitle("Predicting Digits with CNN")
    self.pbTrain.clicked.connect(self.load_train)
    self.pbClear.clicked.connect(self.clear_canvas)
    self.pbSave.clicked.connect(self.save_canvas)
    self.pbPredict.clicked.connect(self.classify_handwriting)

Step 21: Run identify_digit.py and draw something like digit, the click on PREDICT, and you will get the predicted digit and its accuracy as shown in Figure below.

Step 22: Click on CLEAR button and redraw another digit. You wil get the predicted digit and its accuracy as shown in Figure below.

You can see that in both cases the CNN model mistakenly predict the digit and its accuracy is very low. You will improve this by preprocessing the input image in the next tutorial.

Below is the full script of  identify_digit.py so far:

#identify_digit.py
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.uic import loadUi
from matplotlib.backends.backend_qt5agg import (NavigationToolbar2QT as NavigationToolbar)
from widget_paint import widget_paint
from PIL import ImageGrab, Image
import numpy as np
from tensorflow.keras.models import load_model

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.datasets import mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from tensorflow.keras import backend as K

class Identify_Digit(QMainWindow):   
    def __init__(self):       
        QMainWindow.__init__(self)
        loadUi("predict_mnist.ui",self)
        self.setWindowTitle("Predicting Digits with CNN")
        self.pbTrain.clicked.connect(self.load_train)
        self.pbClear.clicked.connect(self.clear_canvas)
        self.pbSave.clicked.connect(self.save_canvas)
        self.pbPredict.clicked.connect(self.classify_handwriting)

    def load_train(self):
        #Loads data and trains model
        # the data, split between train and test sets
        (x_train, y_train), (x_test, y_test) = mnist.load_data()
        print(x_train.shape, y_train.shape)

        #Preprocesses the data
        x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
        x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
        input_shape = (28, 28, 1)

        # convert class vectors to binary class matrices
        y_train = keras.utils.to_categorical(y_train,\
            num_classes=None)
        y_test = keras.utils.to_categorical(y_test, \
            num_classes=None)
        x_train = x_train.astype('float32')
        x_test = x_test.astype('float32')
        x_train /= 255
        x_test /= 255
        print('x_train shape:', x_train.shape)
        print(x_train.shape[0], 'train samples')
        print(x_test.shape[0], 'test samples')

        #Creates the model
        batch_size = 128
        num_classes = 10
        epochs = 10

        model = Sequential()
        model.add(Conv2D(32, kernel_size=(3, 3),\
            activation='relu',input_shape=input_shape))
        model.add(Conv2D(64, (3, 3), activation='relu'))
        model.add(MaxPooling2D(pool_size=(2, 2)))
        model.add(Dropout(0.25))
        model.add(Flatten())
        model.add(Dense(256, activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(num_classes, activation='softmax'))

        model.compile(loss=keras.losses.categorical_crossentropy,\
            optimizer=keras.optimizers.Adadelta(),\
            metrics=['accuracy'])

        #Trains the model
        hist = model.fit(x_train, y_train,\
            batch_size=batch_size,epochs=epochs,verbose=1,\
            validation_data=(x_test, y_test))
        print("The model has successfully trained")

        model.save('mnist.h5')
        print("Saving the model as mnist.h5")

        #Evaluates the model
        score = model.evaluate(x_test, y_test, verbose=0)
        print('Test loss:', score[0])
        print('Test accuracy:', score[1])
        
        pbTrain.setEnabled(False)
        
	# method for clearing every thing on canvas
    def clear_canvas(self):
        self.widgetDigit = widget_paint(self)
        self.widgetDigit.clear()
        self.widgetDigit.show()

	# method for saving image on canvas
    def save_canvas(self):
        self.widgetDigit.save()

    def predict_digit(self, img):
        model = load_model('mnist.h5')
        
        #resize image to 28x28 pixels
        img = img.resize((28,28))
    
        #convert rgb to grayscale
        img = img.convert('L')
        img = np.array(img)
    
        #reshaping to support our model input and normalizing
        img = img.reshape(1,28,28,1)
        img = img/255.0        
        
        #predicting the class
        res = model.predict([img])[0]
        return np.argmax(res), max(res)

    def classify_handwriting(self):
        self.widgetDigit.return_image()        
        im = Image.open("test.png")
        
        digit, acc = self.predict_digit(im)
        self.labelPredict1.setText('Predicted= '+str(digit))
        self.labelPredict2.setText('Accuracy= '+ str(int(acc*100)))
        
if __name__ == '__main__':
    import sys
    app = QApplication(sys.argv)
    ex = Identify_Digit()
    ex.show()
    sys.exit(app.exec_())

Deep Learning to Predict Handwritten Digits with Python GUI: Part 2