how can I parse audio raw data recorder with gstreamer?

Question

I'm writing a C application that uses gstreamer to record audio from a microphone. I want to be able to parse that audio and show a visualization of that audio.

I have the following code:

#include <gst/gst.h>
#include <glib.h>


static gboolean
bus_call (GstBus     *bus,
      GstMessage *msg,
      gpointer    data)
{
GMainLoop *loop = (GMainLoop *) data;

switch (GST_MESSAGE_TYPE (msg)) {

case GST_MESSAGE_EOS:
  g_print ("End of stream\n");
  g_main_loop_quit (loop);
  break;

case GST_MESSAGE_ERROR: {
  gchar  *debug;
  GError *error;

  gst_message_parse_error (msg, &error, &debug);
  g_free (debug);

  g_printerr ("Error: %s\n", error->message);
  g_error_free (error);

  g_main_loop_quit (loop);
  break;
}
default:
  break;
}

return TRUE;
}


void create_loop()
{
GMainLoop *loop;

GstElement *pipeline, *source, *sink;
GstBus *bus;
guint bus_watch_id;

 /* Initialisation */

loop = g_main_loop_new (NULL, FALSE);



/* Create gstreamer elements */
pipeline = gst_pipeline_new ("audio-player");
source   = gst_element_factory_make ("alsasrc",       "alsa-source");
sink     = gst_element_factory_make ("autoaudiosink", "audio-output");

if (!pipeline || !source  || !sink) {
 g_printerr ("One element could not be created. Exiting.\n");
 return;
}

g_object_set (G_OBJECT(source),"device","hw:3,0",NULL);

/* we add a message handler */
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
gst_object_unref (bus);

gst_bin_add_many (GST_BIN (pipeline),
                source,  sink, NULL);

gst_element_link (source, sink);
gst_element_set_state (pipeline, GST_STATE_PLAYING);


/* Iterate */
g_print ("Running...\n");
g_main_loop_run (loop);


/* Out of the main loop, clean up nicely */
g_print ("Returned, stopping playback\n");
gst_element_set_state (pipeline, GST_STATE_NULL);

g_print ("Deleting pipeline\n");
gst_object_unref (GST_OBJECT (pipeline));
g_source_remove (bus_watch_id);
g_main_loop_unref (loop);

}

int main(int argc, char** argv) {
  gst_init(&argc,&argv);
  create_loop();
  return 0;
}

as you can see in my code i create an alsasrc and autoaudiosink. I tested and I can listen properly to that device.

how can I write something in the middle that parses the data in order to create visualization.

any information regarding the issue would be greatly appreciated.

Answer 1

appsink element allows you to get data from the pipeline.

You have three options:

• pull: start pipeline, then get samples with gst_app_sink_pull_sample
• push with signals: enable new-sample signal emission with gst_app_src_set_emit_signals, and subscribe to that signals
• push with callbacks: register callback on new-sample with gst_app_sink_set_callbacks

I think that the third option is the easiest and more efficient.

So, just replace autoaudiosink with appsink, register callback and handle your data in it.

You can read a bit about appsrc and appsink in the manual.

Answer 2

You need to create a "plugin" or a pad for your gsteamer application. There is a complete description on how to create a pad : http://gstreamer.freedesktop.org/data/doc/gstreamer/head/pwg/html/chapter-building-boiler.html#section-boiler-source

In most case you need to create a class using gobject. https://developer.gnome.org/gobject/stable/

Here's how : https://developer.gnome.org/gobject/stable/chapter-gobject.html

And you need at least :

/* Definition of structure storing data for this element. */
typedef struct _GstMyFilter {
  GstElement element;

  GstPad *sinkpad, *srcpad;

  gboolean silent;

} GstMyFilter;


static GstStaticPadTemplate sink_factory =
GST_STATIC_PAD_TEMPLATE (
  "sink",
  GST_PAD_SINK,
  GST_PAD_ALWAYS,
  GST_STATIC_CAPS ("ANY")
);

//... likewise for your src

static void
gst_my_filter_class_init (GstMyFilterClass * klass)
{
  GstElementClass *element_class = GST_ELEMENT_CLASS (klass);

  //...
  gst_element_class_set_static_metadata (element_klass,
    "An example plugin",
    "Example/FirstExample",
    "Shows the basic structure of a plugin",
        "your name <your.name@your.isp>");

  gst_element_class_add_pad_template (element_class,
    gst_static_pad_template_get (&src_factory));
  gst_element_class_add_pad_template (element_class,
    gst_static_pad_template_get (&sink_factory));
}

Your also need description for your plugin

static gboolean
plugin_init (GstPlugin *plugin)
{
  return gst_element_register (plugin, "my_filter",
                   GST_RANK_NONE,
                   GST_TYPE_MY_FILTER);
}

GST_PLUGIN_DEFINE (
  GST_VERSION_MAJOR,
  GST_VERSION_MINOR,
  my_filter,
  "My filter plugin",
  plugin_init,
  VERSION,
  "LGPL",
  "GStreamer",
  "http://gstreamer.net/"
)


static void
gst_my_filter_init (GstMyFilter *filter)
{
  /* pad through which data comes in to the element */
  filter->sinkpad = gst_pad_new_from_static_template (
    &sink_template, "sink");
  /* pads are configured here with gst_pad_set_*_function () */


  gst_element_add_pad (GST_ELEMENT (filter), filter->sinkpad);

  /* pad through which data goes out of the element */
  filter->srcpad = gst_pad_new_from_static_template (
    &src_template, "src");
  /* pads are configured here with gst_pad_set_*_function () */

  gst_element_add_pad (GST_ELEMENT (filter), filter->srcpad);

  /* configure chain function on the pad before adding
   * the pad to the element */
  gst_pad_set_chain_function (filter->sinkpad,
      gst_my_filter_chain);

  /* properties initial value */
  filter->silent = FALSE;
}

and your the magic is operated the chain

static GstFlowReturn
gst_my_filter_chain (GstPad    *pad,
                     GstObject *parent,
             GstBuffer *buf)
{
  GstMyFilter *filter = GST_MY_FILTER (parent);

  if (!filter->silent)
    g_print ("Have data of size %" G_GSIZE_FORMAT" bytes!\n",
        gst_buffer_get_size (buf));

  return gst_pad_push (filter->srcpad, buf);
}

static gboolean
gst_my_filter_sink_event (GstPad    *pad,
                  GstObject *parent,
                  GstEvent  *event)
{
  GstMyFilter *filter = GST_MY_FILTER (parent);

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_CAPS:
      /* we should handle the format here */
      break;
    case GST_EVENT_EOS:
      /* end-of-stream, we should close down all stream leftovers here */
      gst_my_filter_stop_processing (filter);
      break;
    default:
      break;
  }

  return gst_pad_event_default (pad, parent, event);
}

static GstFlowReturn
gst_my_filter_chain (GstPad    *pad,
             GstObject *parent,
             GstBuffer *buf)
{
  GstMyFilter *filter = GST_MY_FILTER (parent);
  GstBuffer *outbuf;

  outbuf = gst_my_filter_process_data (filter, buf);
  gst_buffer_unref (buf);
  if (!outbuf) {
    /* something went wrong - signal an error */
    GST_ELEMENT_ERROR (GST_ELEMENT (filter), STREAM, FAILED, (NULL), (NULL));
    return GST_FLOW_ERROR;
  }

  return gst_pad_push (filter->srcpad, outbuf);
}

use case :

//... your code and elements init...

//Creating your filter
filter = gst_element_factory_make ("my_filter", "my_filter");

//Adding it
gst_bin_add_many (GST_BIN (pipeline), source, filter, sink, NULL);

//Linking it
if (!gst_element_link_many (source, filter, sink, NULL)) {
    g_print ("Failed to link one or more elements!\n");
    return -1;
}

//... here the rest of your code

Bonus * There is a simple example : http://siilo.dyndns.org/wiki/index.php/Creating_a_Plugin_template