EmguCV - Face Recognition - 'Object reference not set' exception when using training set from Microsoft Access Database

StackOverflow https://stackoverflow.com/questions/16453792

Question

I've been developing a face recognition application using EmguCV (C#). I got the whole thing working okay if I store the face images (training set) in simple windows folder. But, after I tried to migrate the face images to be stored in a Microsoft Access database, an 'object reference not set to an instance of an object' exception message often occurs (not always, but most of the time) when the application tries to recognize a face from the video feed.

Funny thing is, the recognition actually still works okay if the exception happens to not occur.

Here is the snippet of the code of my program, using windows folder and database:

Reading the stored images from a Windows Folder

private void FaceRecognition_Load(object sender, EventArgs e)
    {
        //if capture is not created, create it now
        if (capture == null)
        {
            try
            {
                capture = new Capture();
            }
            catch (NullReferenceException excpt)
            {
                MessageBox.Show(excpt.Message);
            }
        }

        if (capture != null)
        {
            if (captureInProgress)
            {  
                Application.Idle -= ProcessFrame;
            }
            else
            {
                Application.Idle += ProcessFrame;
            }

            captureInProgress = !captureInProgress;
        }

        #endregion
        {
            // adjust path to find your xml at loading
            haar = new HaarCascade("haarcascade_frontalface_default.xml");

            try
            {
                //Load of previus trainned faces and labels for each image
                string Labelsinfo = File.ReadAllText(Application.StartupPath + "\\TrainedFaces\\TrainedLabels.txt");
                string[] Labels = Labelsinfo.Split('%');
                NumLabels = Convert.ToInt16(Labels[0]);
                ContTrain = NumLabels;
                string LoadFaces;

                for (int tf = 1; tf < NumLabels + 1; tf++)
                {
                    LoadFaces = "face" + tf + ".bmp";
                    trainingImages.Add(new Image<Gray, byte>(Application.StartupPath + "\\TrainedFaces\\" + LoadFaces));
                    labels.Add(Labels[tf]);
                }

            }
            catch (Exception error)
            {
                //MessageBox.Show(e.ToString());
                MessageBox.Show("Nothing in binary database, please add at least a face(Simply train the prototype with the Add Face Button).", "Triained faces load", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
            }
        }
    }

Reading the stored images from a Microsoft Access Database

private void connectToDatabase()
    {
        DBConnection.ConnectionString = @"Provider=Microsoft.Jet.OLEDB.4.0;Data Source=FacesDatabase.mdb";
        DBConnection.Open();
        dataAdapter = new OleDbDataAdapter("Select * from TrainingSet1", DBConnection);
        dataAdapter.Fill(localDataTable);

        if (localDataTable.Rows.Count != 0)
        {
            numOfRows = localDataTable.Rows.Count;
        }
    }

private void FaceRecognition_Load(object sender, EventArgs e)
    {
        //if capture is not created, create it now
        if (capture == null)
        {
            try
            {
                capture = new Capture();
            }
            catch (NullReferenceException excpt)
            {
                MessageBox.Show(excpt.Message);
            }
        }

        if (capture != null)
        {
            if (captureInProgress)
            {  
                Application.Idle -= ProcessFrame;
            }
            else
            {
                Application.Idle += ProcessFrame;
            }

            captureInProgress = !captureInProgress;
        }

        #endregion
        {
            // adjust path to find your xml at loading
            haar = new HaarCascade("haarcascade_frontalface_default.xml");

            connectToDatabase();

            Bitmap bmpImage;

            for (int i = 0; i < numOfRows; i++)
            {
                byte[] fetchedBytes = (byte[])localDataTable.Rows[i]["FaceImage"];
                MemoryStream stream = new MemoryStream(fetchedBytes);
                bmpImage = new Bitmap(stream);
                trainingImages.Add(new Emgu.CV.Image<Gray, Byte>(bmpImage));

                String faceName = (String)localDataTable.Rows[i]["Name"];
                labels.Add(faceName);
            }
       }
   }

The face recognition function that causes the exception (exactly the same both when using windows folder and Access database):

private void ProcessFrame(object sender, EventArgs arg)
    {
        Image<Bgr, Byte> ImageFrame = capture.QueryFrame();

        Image<Gray, byte> grayframe = ImageFrame.Convert<Gray, byte>();

        MinNeighbors = int.Parse(comboBoxMinNeighbors.Text);
        WindowsSize = int.Parse(textBoxWinSiz.Text);
        ScaleIncreaseRate = Double.Parse(comboBoxMinNeighbors.Text);

        var faces = grayframe.DetectHaarCascade(haar, ScaleIncreaseRate, MinNeighbors,
                                        HAAR_DETECTION_TYPE.DO_CANNY_PRUNING,
                                        new Size(WindowsSize, WindowsSize))[0];

        if (faces.Length > 0) 
        {
            Bitmap BmpInput = grayframe.ToBitmap();

            Graphics FaceCanvas;

            foreach (var face in faces)
            {
                t = t + 1;
                result = ImageFrame.Copy(face.rect).Convert<Gray, byte>().Resize(100, 100, Emgu.CV.CvEnum.INTER.CV_INTER_CUBIC);

                ImageFrame.Draw(face.rect, new Bgr(Color.Red), 2);

                ExtractedFace = new Bitmap(face.rect.Width, face.rect.Height);

                FaceCanvas = Graphics.FromImage(ExtractedFace);

                FaceCanvas.DrawImage(BmpInput, 0, 0, face.rect, GraphicsUnit.Pixel);

                ImageFrame.Draw(face.rect, new Bgr(Color.Red), 2);

                if (trainingImages.ToArray().Length != 0)
                {

                    MCvTermCriteria termCrit = new MCvTermCriteria(ContTrain, 0.001);

                    EigenObjectRecognizer recognizer = new EigenObjectRecognizer(
                        trainingImages.ToArray(),
                        labels.ToArray(),
                        3000,
                        ref termCrit);
                    try
                    {
                        name = recognizer.Recognize(result).Label; 
                    }
                    catch (Exception error)
                    {
                        MessageBox.Show(error.ToString());
                    }

                    ImageFrame.Draw(name, ref font, new Point(face.rect.X - 2, face.rect.Y - 2), new Bgr(Color.LightGreen));
                }

            }
        }
        CamImageBox.Image = ImageFrame;
    }

Here is the screenshot of the exception message: http://i.imgur.com/DvAhABK.jpg

Line 146 where the exception occurs is this line of the ProcessFrame function:

name = recognizer.Recognize(result).Label;

I tried searching for similar problems in the internet, and found these: 'Object reference not set to instance of an object' error when trying to upload image to database Object reference not set to an instance of an object #5 C# Error 'Object Reference Not Set To An Instance Of An Object' C#, "Object reference not set to an instance of an object." error

Most of them suggests to check if any of the involved variable is null. I've checked the involved variable, and indeed the exception occurs when the recognizer.Recognize(result) statement returns null.

So my question is, why does that statement often return null when I use training images from the database, while it never returns null when I use training images from windows folder?

Was it helpful?

Solution

Check your fetchedBytes array to see if you are consistently getting just a stream of bytes representing a BMP image (starting with 0x42 0x4D), or if there may be "other stuff" in there, too.

Depending on how the BMP data was inserted into the Access database it may contain an OLE "wrapper". For example, an 8x8 24-bit BMP image of pure red is saved by MSPAINT.EXE like this

bmpDump.png

If I copy that file and paste it into a Bound Object Frame in an Access form then Access wraps the BMP data in some "OLE stuff" before writing it to the table. Later, if I try to retrieve the BMP image via code, using something like this...

Sub oleDumpTest()
    Dim rst As ADODB.Recordset, ads As ADODB.Stream
    Set rst = New ADODB.Recordset
    rst.Open "SELECT * FROM TrainingSet1 WHERE ID = 1", Application.CurrentProject.Connection
    Set ads = New ADODB.Stream
    ads.Type = adTypeBinary
    ads.Open
    ads.Write rst("FaceImage").Value
    rst.Close
    Set rst = Nothing
    ads.SaveToFile "C:\Users\Gord\Pictures\oleDump_red."
    ads.Close
    Set ads = Nothing
End Sub

...then the resulting file also contains the OLE "wrapper"...

oleDump.png

...and obviously is not a valid stand-alone BMP file. If I rename that file to give it a .bmp extension and try to open it in Paint, I get

paintError.png

So maybe (some of) the [FaceImage] objects in your database are not raw BMP data, and perhaps the other software is rejecting them (or simply not able to understand them).

Edit

Another possible issue is that when you get the images from files in a folder you hand the Image object a string containing the file path...

trainingImages.Add(new Image<Gray, byte>(Application.StartupPath + "\\TrainedFaces\\" + LoadFaces));

...but when you try to retrieve the images from the database you hand the same object a Bitmap object

MemoryStream stream = new MemoryStream(fetchedBytes);
bmpImage = new Bitmap(stream);
trainingImages.Add(new Emgu.CV.Image<Gray, Byte>(bmpImage));

I have no way of knowing whether the Emgu.CV.Image object might behave differently depending on the type of object it is given, but a quick+dirty workaround might be to write bmpImage to a temporary file, hand trainingImages.Add the path to that file, and then delete the file.

OTHER TIPS

Finally made it!! just one more day of coding helped me to got the problem solved:

public void ProcessRequest(HttpContext context)
{
    _httpContext = context;
    var imageid = context.Request.QueryString["Image"];
    if (imageid == null || imageid == "")
    {
        imageid = "1";
    }


    using (WebClient wc = new WebClient())
    {
        // Handler retrieves the image from database and load it on the stream
        using (Stream s = wc.OpenRead("http://mypageurl/Image.ashx?Image=" + imageid))
        {
            using (Bitmap bmp = new Bitmap(s))
            {
                AddFace(bmp);
            }
        }
    }

}

public void AddFace(Bitmap image)
{
    var faceImage = DetectFace(image);
    if (faceImage != null)
    {
        var stream = new MemoryStream();
        faceImage.Save(stream, ImageFormat.Bmp);
        stream.Position = 0;
        byte[] data = new byte[stream.Length];
        stream.Read(data, 0, (int)stream.Length);

        _httpContext.Response.Clear();
        _httpContext.Response.ContentType = "image/jpeg";
        _httpContext.Response.BinaryWrite(data);
    }
}

private Bitmap DetectFace(Bitmap faceImage)
{
    var image = new Image<Bgr, byte>(faceImage);
    var gray = image.Convert<Gray, Byte>();
    string filePath = HttpContext.Current.Server.MapPath("haarcascade_frontalface_default.xml");
    var face = new HaarCascade(filePath);
    MCvAvgComp[][] facesDetected = gray.DetectHaarCascade(face, 1.1, 10, HAAR_DETECTION_TYPE.DO_CANNY_PRUNING, new Size(20, 20));
    Image<Gray, byte> result = null;

    foreach (MCvAvgComp f in facesDetected[0])
    {
        //draw the face detected in the 0th (gray) channel with blue color
        image.Draw(f.rect, new Bgr(Color.Blue), 2);
        result = image.Copy(f.rect).Convert<Gray, byte>();
        break;
    }

    if (result != null)
    {
        result = result.Resize(200, 200, INTER.CV_INTER_CUBIC);
        return result.Bitmap;
    }


   return null;
}

public bool IsReusable
{
    get { return false; }
}

I couldnt make it work from reading a direct Stream from the the Database where the images are located but your workaround, saving the images to a local folder, worked for me, thx a lot for sharing.Here's my demo page where you load files from DB: http://www.edatasoluciones.com/FaceDetection/FaceDataBase

Licensed under: CC-BY-SA with attribution
Not affiliated with StackOverflow
scroll top