Is JMS the answer to the need for a persistent blocking queue?

Question

I'm creating a library that consists of a Log4J appender that asynchronously sends events to a remote server. When a log statement is made, the appender will asynchronously record the event into a local queue which a pool of consumers will then retrieve and send to the remote.

The completely in-memory solution would be to create a BlockingQueue which would handle the concurrency issue. However, I'd like for the queue to be persisted so that if the remote server is not available I don't grow the queue unbounded or start to discard messages in the case of a bounded queue.

I was thinking of using an embedded H2 database to store the events locally and then use a polling mechanism to retrieve events and send to the remote. I would much rather use a BlockingQueue than to poll a database table.

Is JMS the answer?

EDIT:

If JMS is the answer, and it seems to be going that way, does anyone have recommendations on a lightweight, embeddable JMS solution that can be configured to only accept messages in-process? In other words, I do not want to, and possibly will not be allowed to, open up a TCP socket on which to listen.

EDIT:

I've got ActiveMQ embedded now and it seems to be working. Thanks all.

Answer 1

You could use JMS to asynchronously send messages to a remote machine (assuming it can receive them of course), Log4j has a JMS Appender you can use for this.

Answer 2

Bob Lee open sourced a very simple disk backed queue a while back, https://github.com/square/retrofit/blob/master/modules/android/src/retrofit/io/QueueFile.java -- may be helpful, and is certainly a lot easier to introduce than JMS if you can accept local durability.

This class is standalone -- it can be copied and pasted.

Answer 3

You can definitely use JMS for this purpose. As far as I understand you are using the Log4J JMS appender. This component sends messages to pre-configured JMS destination (typically queue). You can configure this queue to be persisted. In this case all messages inserted into the queue will be automatically stored in some persisted store (typically database.). Unfortunately this configuration is vendor specific (depends on the JMS vendor), but usually is very simple. Please refer to the documentation of you JMS provider.

Answer 4

See if this works

This code should work for you - its an in memory persistent blocking queue - needs some file tuning but should work

       package test;

     import java.io.BufferedReader;
     import java.io.BufferedWriter;
     import java.io.File;
     import java.io.FileReader;
     import java.io.FileWriter;
     import java.io.IOException;
     import java.util.ArrayList;
     import java.util.Collections;
     import java.util.LinkedList;
     import java.util.List;

     public class BlockingQueue {

    //private static Long maxInMenorySize = 1L;
    private static Long minFlushSize = 3L;

    private static String baseDirectory = "/test/code/cache/";
    private static String fileNameFormat = "Table-";

    private static String  currentWriteFile = "";

    private static List<Object>  currentQueue = new LinkedList<Object>();
    private static List<Object>  lastQueue = new LinkedList<Object>();

    static{
        try {
            load();
        } catch (IOException e) {
            System.out.println("Unable To Load");
            e.printStackTrace();
        }
    }

    private static void load() throws IOException{
        File baseLocation = new File(baseDirectory);
        List<String> fileList = new ArrayList<String>();

        for(File entry : baseLocation.listFiles()){
            if(!entry.isDirectory() && entry.getName().contains(fileNameFormat)){
                fileList.add(entry.getAbsolutePath());
            }
        }

        Collections.sort(fileList);

        if(fileList.size()==0){
            //currentQueue = lastQueue = new ArrayList<Object>();
            currentWriteFile = baseDirectory + "Table-1";
            BufferedWriter writer = new BufferedWriter(new FileWriter(currentWriteFile));
            while (!lastQueue.isEmpty()){
                writer.write(lastQueue.get(0).toString()+ "\n");
                lastQueue.remove(0);
            }
            writer.close();
        }else{
            if(fileList.size()>0){
                    BufferedReader reader = new BufferedReader(new FileReader(fileList.get(0)));
                    String line=null;
                    while ((line=reader.readLine())!=null){
                        currentQueue.add(line);
                    }
                    reader.close();
                    File toDelete = new File(fileList.get(0));
                    toDelete.delete();
            }

            if(fileList.size()>0){
                BufferedReader reader = new BufferedReader(new FileReader(fileList.get(fileList.size()-1)));
                currentWriteFile = fileList.get(fileList.size()-1);
                String line=null;
                while ((line=reader.readLine())!=null){
                    lastQueue.add(line);
                }
                reader.close();
                //lastFileNameIndex=Long.parseLong(fileList.get(fileList.size()).substring(6, 9));
            }
        }

    }

    private void loadFirst() throws IOException{
        File baseLocation = new File(baseDirectory);
        List<String> fileList = new ArrayList<String>();

        for(File entry : baseLocation.listFiles()){
            if(!entry.isDirectory() && entry.getName().contains(fileNameFormat)){
                fileList.add(entry.getAbsolutePath());
            }
        }

        Collections.sort(fileList);

        if(fileList.size()>0){
                BufferedReader reader = new BufferedReader(new FileReader(fileList.get(0)));
                String line=null;
                while ((line=reader.readLine())!=null){
                    currentQueue.add(line);
                }
                reader.close();
                File toDelete = new File(fileList.get(0));
                toDelete.delete();
        }
    }

    public Object pop(){
        if(currentQueue.size()>0)
            return  currentQueue.remove(0);

        if(currentQueue.size()==0){
            try {
                loadFirst();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }

        if(currentQueue.size()>0)
            return  currentQueue.remove(0);
        else
            return null;
    }

    public synchronized Object waitTillPop() throws InterruptedException{
        if(currentQueue.size()==0){
            try {
                loadFirst();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            if(currentQueue.size()==0)
                wait();
        }
        return currentQueue.remove(0);
    }

    public synchronized void push(Object data) throws IOException{
        lastQueue.add(data);
        this.notifyAll();
        if(lastQueue.size()>=minFlushSize){
            BufferedWriter writer = new BufferedWriter(new FileWriter(currentWriteFile));
            while (!lastQueue.isEmpty()){
                writer.write(lastQueue.get(0).toString() + "\n");
                lastQueue.remove(0);
            }
            writer.close();

            currentWriteFile  = currentWriteFile.substring(0,currentWriteFile.indexOf("-")+1) + 
                    (Integer.parseInt(currentWriteFile.substring(currentWriteFile.indexOf("-")+1,currentWriteFile.length())) + 1);
        }
    }

    public static void main(String[] args) {
        try {
            BlockingQueue bq = new BlockingQueue();

            for(int i =0 ; i<=8 ; i++){
                bq.push(""+i);
            }

            System.out.println(bq.pop());
            System.out.println(bq.pop());
            System.out.println(bq.pop());

            System.out.println(bq.waitTillPop());
            System.out.println(bq.waitTillPop());
            System.out.println(bq.waitTillPop());
            System.out.println(bq.waitTillPop());



        } catch (Exception e) {
            e.printStackTrace();
        }
    }