How to plot a waveform using PortAudio in Java (jpab with Processing)?

Question

I need to access multiple audio inputs in Java so first consulted SO and found this answer and deviced to use PortAudio Java bindings (jpab). Unfortunately I found little and outdated documentation.

With what I've found I tried this using Processing in eclipse:

import java.nio.ByteBuffer;

import org.jpab.Callback;
import org.jpab.Device;
import org.jpab.PortAudio;
import org.jpab.PortAudioException;
import org.jpab.Stream;
import org.jpab.StreamConfiguration;
import org.jpab.StreamConfiguration.Mode;
import org.jpab.StreamConfiguration.SampleFormat;


import processing.core.PApplet;


public class PortAudioPlot extends PApplet implements Callback {

    float min = 1000000,max = 0;

    public void setup(){
        try {
            PortAudio.initialize();
            for(Device d : PortAudio.getDevices()) println(d);

            Device d = PortAudio.getDevices().get(1);// Microphone (Realtek High Definition Audio)
            if(d.getMaxInputChannels() > 0){
                println(d.getName());
                StreamConfiguration sc = new StreamConfiguration();
                sc.setInputDevice(d);
                sc.setInputFormat(SampleFormat.SIGNED_INTEGER_16);
                sc.setMode(Mode.INPUT_ONLY);
                sc.setSampleRate(44100);
                sc.setInputChannels(d.getMaxInputChannels());
                PortAudio.createStream(sc, this, new Runnable() {
                    public void run() {
                        try {
                            PortAudio.terminate();
                        } catch (PortAudioException ignore) { ignore.printStackTrace(); }
                    }
                }).start();
            }
        } catch (PortAudioException e) {
            e.printStackTrace();
        }
    }
    public void draw(){
        if(keyPressed && key == 's') saveFrame(dataPath("frame-####.jpg"));
    }
    public void stop(){
        try {
            PortAudio.terminate();
        } catch (PortAudioException e) {
            e.printStackTrace();
        }
        super.stop();
    }

    public static void main(String[] args) {
        PApplet.main(PortAudioPlot.class.getSimpleName());
    }
    @Override
    public State callback(ByteBuffer in, ByteBuffer out) {
        int size = in.capacity();
        println("in size: " + size + " min: " + min + " max: " + max);
        background(255);
        beginShape(LINES);
        for (int i = 0; i < size; i++) {
            float v = in.getFloat(i);
            if(!Float.isNaN(v) && v != Float.POSITIVE_INFINITY && v != Float.NEGATIVE_INFINITY){
                float x = (float)i/size * width;
                float y = (height * .5f) + (v * .5f);
                if(v < min) min = v;
                if(v > max) max = v;
                vertex(x,y);
            }
        }
        endShape();
        return State.ABORTED;
    }

}

I started with the mic 1st and I think I'm getting close as I can seem some values, but I'm not 100% sure I'm traversing the input ByteBuffer correctly.

What is the correct way to access values and plot a waveform from an audio input using jpab ?

I've updated the code a wee bit and managed to get something closer to a plot, but I'm still in the dark. What are the correct min/max ranges for floats read from the input ByteBuffer ? Am I using it the right way ?

Here's a quick preview of what I've got:

I've also uploaded the eclipse project here. It's using the prebuilt Windows x86 PortAudio binaries.

Another update: i was advised that the values should be from -1.0 to 1.0 and adjusted my code to map/clamp for this, but I'm not sure if this true. Here is an updated example:

import java.nio.ByteBuffer;
import java.util.Arrays;

import org.jpab.Callback;
import org.jpab.Device;
import org.jpab.PortAudio;
import org.jpab.PortAudioException;
import org.jpab.Stream;
import org.jpab.StreamConfiguration;
import org.jpab.StreamConfiguration.Mode;
import org.jpab.StreamConfiguration.SampleFormat;


import processing.core.PApplet;


public class PortAudioPlot extends PApplet implements Callback {

    int[] pix;
    int hh;//half height
    int py;//y for each channel plot
    int numChannels;
    int pad = 5;

    public void setup(){
        try {
            colorMode(HSB,360,100,100);
            hh = height/2;
            pix = new int[width*height];
            PortAudio.initialize();
            for(Device d : PortAudio.getDevices()) println(d);

            Device d = PortAudio.getDevices().get(1);// Microphone (Realtek High Definition Audio)
            numChannels = d.getMaxInputChannels();
            py = height / numChannels;
            if(numChannels > 0){
                println(d.getName()+" sr:" + d.getDefaultSampleRate());
                StreamConfiguration sc = new StreamConfiguration();
                sc.setInputLatency(d.getDefaultLowInputLatency());
                sc.setInputDevice(d);
                sc.setInputFormat(SampleFormat.SIGNED_INTEGER_16);
                sc.setMode(Mode.INPUT_ONLY);
                sc.setSampleRate(d.getDefaultSampleRate());
                sc.setInputChannels(numChannels);
                PortAudio.createStream(sc, this, new Runnable() {
                    public void run() {
                        try {
                            PortAudio.terminate();
                        } catch (PortAudioException ignore) { ignore.printStackTrace(); }
                    }
                }).start();
            }
        } catch (PortAudioException e) {
            e.printStackTrace();
        }
    }
    public void draw(){
        loadPixels();
        arrayCopy(pix, pixels);
        updatePixels();
        if(keyPressed && key == 's') saveFrame(dataPath("frame-####.jpg"));
    }
    public void stop(){
        try {
            PortAudio.terminate();
        } catch (PortAudioException e) {
            e.printStackTrace();
        }
        super.stop();
    }

    public static void main(String[] args) {
        PApplet.main(PortAudioPlot.class.getSimpleName());
    }
    @Override
    public State callback(ByteBuffer in, ByteBuffer out) {
        int size = in.capacity();
        println("in size: " + size);
        Arrays.fill(pix, color(0,0,100));
        for (int i = 0; i < width; i++) {
            int ch = i%numChannels;//channel id
            int sy = py * ch;//channel plot y starting position
            int minY = sy+pad;//min y for min input value
            int maxY = (sy*2)-pad;//min y for min input value
            int buffIndex = i * size / width;//map i(x pixel index) to buffer index
            float v = in.getFloat(buffIndex);
            if(!Float.isNaN(v) && v != Float.POSITIVE_INFINITY && v != Float.NEGATIVE_INFINITY){
                int vOffset = constrain((int)map(v,-1.0f,1.0f,minY,maxY),minY,maxY);
                pix[vOffset * height + i] = color(map(ch,0,numChannels,0,360),100,50);
            }
        }
        return State.RUNNING;
    }

}

I've also noticed that the input ByteBuffer count changes when I setup latency.

And another confusing thing I noticed: JPAB is not the same as jportaudio, although most the API is similar, excepting createStream(jpab)/openStream(jportaudio). I haven't found a compiled version of jportaudio and haven't managed to compile it myself on Windows so far.

Any clues on how I can continue ?

Answer 1

The end goal is to access multiple audio inputs and this route at this point in time doesn't seem to lead anywhere.

The simplest solution which I've tested on Windows and OSX, is easy to setup and works in plain Java but quite nicely in Processing as well is using Beads which can connect to JACK. See this thread for more details, especially the later part about JNAJack (JJack is no longer maintained). I've used this version of Beads(download link) and JNA(download link).

Here's a basic code sample I used to test:

import java.util.Arrays;

import org.jaudiolibs.beads.AudioServerIO;

import net.beadsproject.beads.core.AudioContext;
import net.beadsproject.beads.core.AudioIO;
import net.beadsproject.beads.core.UGen;
import net.beadsproject.beads.ugens.Gain;
import processing.core.PApplet;


public class BeadsJNA extends PApplet {

    AudioContext ac;

    public void setup(){
        ac = new AudioContext(new AudioServerIO.Jack(),512,AudioContext.defaultAudioFormat(4,2));//control number of ins(4) and outs(2)

        UGen microphoneIn = ac.getAudioInput();
        Gain g = new Gain(ac, 1, 0.5f);
        g.addInput(microphoneIn);
        ac.out.addInput(g);

        println("no. of inputs:  " + ac.getAudioInput().getOuts()); 

        ac.start();
    }
    public void draw(){
        loadPixels();
      Arrays.fill(pixels, color(0));

      for(int i = 0; i < width; i++)
      {
        int buffIndex = i * ac.getBufferSize() / width;
        int vOffset = (int)((1 + ac.out.getValue(0, buffIndex)) * height / 2);
        pixels[vOffset * height + i] = color(255);
      }
      updatePixels(); 
    }

    public static void main(String[] args) {
        PApplet.main(BeadsJNA.class.getSimpleName());
    }

}

This worked for me at this stage in time so is a valid answer until someone will share an easy way to use jpab/jportaudio on Windows to plot the waveform from an input.