Java: opencv & openni

Question

I was wondering if anyone has been able to use openni in combination with opencv in java? For example that you get the depth stream in a IplImage etc ... I'm currently trying to do this , but i'm not sure where to start.

If anyone who has done this want to share their knowledge or some code i would be grateful.

My code so far:

/

*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.


/**
 *
 * @author olivierjanssens
 */

package kitouch;

import com.googlecode.javacpp.Loader;
import com.googlecode.javacv.*;
import com.googlecode.javacv.cpp.*;
import static com.googlecode.javacv.cpp.opencv_core.*;
import static com.googlecode.javacv.cpp.opencv_imgproc.*;
import static com.googlecode.javacv.cpp.opencv_calib3d.*;
import static com.googlecode.javacv.cpp.opencv_objdetect.*;
import java.nio.ShortBuffer;
import java.awt.*;
import java.awt.image.*;
import org.OpenNI.*;
import javax.swing.JFrame;

public class KiTouch {

    private Context context;
    private final String SAMPLE_XML_FILE = "/Users/olivierjanssens/Development/Kinect/OpenNI/Samples/Config/SamplesConfig.xml";    
    private OutArg<ScriptNode> scriptNode;
    private DepthGenerator depthGen;
    private BufferedImage bimg;
     int width, height;
    IplImage depthImage;
    private float histogram[];
     private byte[] imgbytes;
    CanvasFrame frame = new CanvasFrame("Some Title");

    public KiTouch() {
        try {
            scriptNode = new OutArg<ScriptNode>();
            context = Context.createFromXmlFile(SAMPLE_XML_FILE, scriptNode);

            depthGen = DepthGenerator.create(context);
            DepthMetaData depthMD = depthGen.getMetaData();

            histogram = new float[10000];
            width = depthMD.getFullXRes();
            height = depthMD.getFullYRes();

            imgbytes = new byte[width*height];

            DataBufferByte dataBuffer = new DataBufferByte(imgbytes, width*height);
            Raster raster = Raster.createPackedRaster(dataBuffer, width, height, 8, null);
            bimg = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY);
            bimg.setData(raster);
            depthImage = IplImage.create(width, height, IPL_DEPTH_8U, 1);

        } catch (GeneralException e) {
            e.printStackTrace();
            System.exit(1);
        }


    }


    private void calcHist(DepthMetaData depthMD)
    {
        // reset
        for (int i = 0; i < histogram.length; ++i)
            histogram[i] = 0;

        ShortBuffer depth = depthMD.getData().createShortBuffer();
        depth.rewind();

        int points = 0;
        while(depth.remaining() > 0)
        {
            short depthVal = depth.get();
            if (depthVal != 0)
            {
                histogram[depthVal]++;
                points++;
            }
        }

        for (int i = 1; i < histogram.length; i++)
        {
            histogram[i] += histogram[i-1];
        }

        if (points > 0)
        {
            for (int i = 1; i < histogram.length; i++)
            {
                histogram[i] = (int)(256 * (1.0f - (histogram[i] / (float)points)));
            }
        }
    }

    public Dimension getPreferredSize() {
        return new Dimension(width, height);
    }

     void updateDepth()
    {
        try {
            DepthMetaData depthMD = depthGen.getMetaData();

            context.waitAnyUpdateAll();

            calcHist(depthMD);
            ShortBuffer depth = depthMD.getData().createShortBuffer();
            depth.rewind();

            while(depth.remaining() > 0)
            {
                int pos = depth.position();
                short pixel = depth.get();
                imgbytes[pos] = (byte)histogram[pixel];
            }
            depthImage.createFrom(bimg);
            frame.showImage(depthImage);

        } catch (GeneralException e) {
            e.printStackTrace();
        }
    }


}

and to call this code :

/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package kitouch;

import org.OpenNI.GeneralException;

/**
 *
 * @author olivierjanssens
 */
public class kiTouchApp {
     public static void main(String s[]) throws GeneralException {
         KiTouch kit = new KiTouch();


          while(true) {
            kit.updateDepth();
        }
    }
}

Though i'm getting a black frame. So it doens't work yet

When i do not initalize the IplImage like i do here but just IplImage depthImage = new IplImage(); i get this error:

Exception in thread "main" java.lang.NullPointerException
at java.awt.image.BufferedImage.<init>(BufferedImage.java:613)
at com.googlecode.javacv.cpp.opencv_core$IplImage.getBufferedImage(opencv_core.java:1005)
at com.googlecode.javacv.cpp.opencv_core$IplImage.getBufferedImage(opencv_core.java:931)
at com.googlecode.javacv.CanvasFrame.showImage(CanvasFrame.java:331)
at kitouch.KiTouch.paint(KiTouch.java:138)
at kitouch.kiTouchApp.main(kiTouchApp.java:21)

Thx in advance !

Answer 1

I did play with OpenNI and Java, but using Processing and the available wrappers(SimpleOpenNI and OpenCV) which for my modest needs at the moment work well. Here's a very basic example:

import hypermedia.video.*;
import SimpleOpenNI.*;

SimpleOpenNI  ni;
OpenCV cv;
PImage user;
void setup()
{
  ni = new SimpleOpenNI(this);
  ni.enableScene();
  background(200,0,0);
  strokeWeight(4);
  size(ni.sceneWidth() , ni.sceneHeight()); 
  cv = new OpenCV( this );
  cv.allocate(width,height);
}
void draw()
{
  //OpenNI
  ni.update();
  user = ni.sceneImage(); 
  //OpenCV
  cv.copy(user);
  cv.blur( OpenCV.BLUR, 17 );
  Blob[] blobs = cv.blobs( width,height, OpenCV.MAX_VERTICES, true, OpenCV.MAX_VERTICES*4 );

  //diplay
  image(cv.image(),0,0);
  //*
  fill(255);
  for(Blob b : blobs){
    beginShape();
      for(java.awt.Point p : b.points) vertex(p.x,p.y);
    endShape(CLOSE);
  }
  //*/
}

Note that this OpenCV wrapper uses OpenCV 1.0 and also you might want to use your own Java classes, rather using the Processing library. In this case try the JavaCV wrapper. Regarding the depth stream, if you look in the org.OpenNI.Samples.SimpleViewer class that comes with OpenNI, you will notice that the depth bytes from OpenNI are written to a BufferedImage, which I imagine can be integrated with OpenCV,etc.

 public SimpleViewer() {

        try {
            scriptNode = new OutArg<ScriptNode>();
            context = Context.createFromXmlFile(SAMPLE_XML_FILE, scriptNode);

            depthGen = DepthGenerator.create(context);
            DepthMetaData depthMD = depthGen.getMetaData();

            histogram = new float[10000];
            width = depthMD.getFullXRes();
            height = depthMD.getFullYRes();

            imgbytes = new byte[width*height];

            DataBufferByte dataBuffer = new DataBufferByte(imgbytes, width*height);
            Raster raster = Raster.createPackedRaster(dataBuffer, width, height, 8, null);
            bimg = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY);
            bimg.setData(raster);

        } catch (GeneralException e) {
            e.printStackTrace();
            System.exit(1);
        }
    }