java making a screen shot preview with JPanel

Question

I'm making a fun little test screen recording program in java, and I want it to have a preview of your screen before you start recording.. but its a very slow and poor method of which I am using, involving capturing an image, saving it, then reading it in through a bufferedimage and drawing that image using Graphics. Its very slow and not useful as a "preview" is there a way to speed up and have a more efficient "previewing system". Here is what I have so far:

import java.awt.BorderLayout;
import java.awt.Color;
import java.awt.Container;
import java.awt.Dimension;
import java.awt.Graphics;
import java.awt.GridLayout;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.Toolkit;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import javax.imageio.ImageIO;
import javax.swing.JFrame;
import javax.swing.JButton;
import javax.swing.JOptionPane;
import javax.swing.JPanel;

public class MainFrame implements ActionListener, Runnable {
    //add frame components
    public static JFrame frame = new JFrame("Screen Caper - v1.0.1");
    JButton start = new JButton("record");
    JButton close = new JButton("Exit");
    JPanel preview = new JPanel();
    public static boolean running = false;
    public static boolean recording = false;
    public static boolean paused = false;
    public static String curDir = System.getProperty("user.dir");
    //get the screen width
    Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize();
    double width = screenSize.getWidth();
    double height = screenSize.getHeight();
    Container a = new Container();
    Container b = new Container();
    public MainFrame() {
        frame.setSize((int)(width) - 80, (int)(height) - 80);
        frame.setLocationRelativeTo(null);
        frame.setResizable(false);
        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        //setup the buttons and JPanel
        a.setLayout(new GridLayout(1, 2));
        a.add(start);
        start.addActionListener(this);
            a.add(close);
        close.addActionListener(this);
        frame.add(a, BorderLayout.NORTH);
        b.setLayout(new GridLayout(1, 2));
        b.add(preview);
        frame.add(b, BorderLayout.CENTER);
        //add anything else
        running = true;
        //set frame to visible
        frame.setVisible(true);
        run();
    }
    public static void main(String[] args) {
        new MainFrame();
    }
    public void run() {
        Graphics g = frame.getGraphics();
        while (running) {
            //draw the preview of the computer screen on the JPanel if its not recording already
            if (!recording && !paused) {
                drawPreview(g);
            }
        }
    }
    public void drawPreview(Graphics g) {
        BufferedImage image;
        try {
            image = new Robot().createScreenCapture(new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()));
            ImageIO.write(image, "png", new File("test.png"));
        } catch (Exception ex) {
            ex.printStackTrace();
        }
        BufferedImage prevIm;
        try {
                prevIm = ImageIO.read(new File("test.png"));
                g.setColor(new Color(0, 0, 0));
                g.fillRect(preview.getX() + 3, preview.getY() + 51, preview.getWidth(), preview.getHeight() + 1);
                g.drawImage(prevIm, preview.getX() + 3, preview.getY() + 51, preview.getX() + preview.getWidth(), preview.getY() + preview.getHeight(), null);
        } catch (IOException ex) {
            ex.printStackTrace();
        }
    }
    public void record(Graphics g) {

    }
    @Override
    public void actionPerformed(ActionEvent event) {
        if (event.getSource().equals(start)) {
            if (!recording) {
                //if the program isn't recording, then start recording
                Graphics g = frame.getGraphics();
                record(g);
                start.setText("Finish");
                recording = true;
                System.out.println("recording...");
            } else {
                //else stop recording


                start.setText("record");
                recording = false;
                System.out.println("done");
            }
        }
        if (event.getSource().equals(close)) {
            paused = true;
        int ans = JOptionPane.showConfirmDialog(null, "Woah there! You're about to quit the application\nAre you sure you want to procced?", "Caution!", JOptionPane.YES_NO_OPTION);
            if (ans == JOptionPane.YES_OPTION) {
                System.exit(0);
            } else if (ans == JOptionPane.NO_OPTION) {
                paused = false;
            }
        }
    }
}

any help is appreciated!

Answer 1

• Don't use getGraphics, this is not how custom painting is done.
• Your run method may simply be running to fast, consider using a javax.swing.Timer instead
• Toolkit.getDefaultToolkit().getScreenSize() only returns the "default" screen and does not take into consideration split screens
• Capturing the screen is a time consuming process and there's not much you can do about reducing it (as it's outside of your control). You "could" setup a series of Threads whose job it was to capture a given section of the desktop. You could also achieve this using SwingWorkers which would make it easier to sync the updates back to the UI...

Take a look at:

• Performing Custom Painting
• Concurrency in Swing
• How to Use Swing Timers

Updated with example

This is a proof of concept only! You should understand what it's trying to do and borrow ideas from it...

You can see the preview window change inside the preview window...kinda freaky...

It tested this on a virtual desktop of 4480x1600.

Basically, it divides the desktop up into a 4x4 grid and starts a Thread for each section. Each thread is responsible for capturing it's own section of the screen.

It also scales that resulting image down and feeds it back to the UI.

I had started with SwingWorkers, but it seems to be hard to be limited to 10 threads. You could reduce the grid size and use SwingWorkers, they tend to be simpler to use and manage then raw Threads.

Each section is given an id, which allows me keep track of what's changed. Technically, you could just add elements to a List, but how do you determine what's new and what's old?

import java.awt.AWTException;
import java.awt.BorderLayout;
import java.awt.Color;
import java.awt.Dimension;
import java.awt.EventQueue;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.GraphicsDevice;
import java.awt.GraphicsEnvironment;
import java.awt.Rectangle;
import java.awt.RenderingHints;
import java.awt.Robot;
import java.awt.Transparency;
import java.awt.image.BufferedImage;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantLock;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import javax.swing.UIManager;
import javax.swing.UnsupportedLookAndFeelException;

public class PreviewDesktop {

    public static void main(String[] args) {
        new PreviewDesktop();
    }

    public PreviewDesktop() {
        EventQueue.invokeLater(new Runnable() {
            @Override
            public void run() {
                try {
                    UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
                } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | UnsupportedLookAndFeelException ex) {
                }

                JFrame frame = new JFrame("Testing");
                frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
                frame.setLayout(new BorderLayout());
                frame.add(new TestPane());
                frame.pack();
                frame.setLocationRelativeTo(null);
                frame.setVisible(true);
            }
        });
    }

    public class TestPane extends JPanel implements Puzzler {

        private Rectangle virtualBounds;
        private double scale;

        private Map<Integer, PuzzlePiece> pieces;

        public TestPane() {
            virtualBounds = getVirtualBounds();
            int columns = 4;
            int rows = 4;
            pieces = new HashMap<>(columns * rows);

            int columnWidth = Math.round(virtualBounds.width / (float) columns);
            int rowHeight = Math.round(virtualBounds.height / (float) rows);

            int id = 0;
            for (int row = 0; row < rows; row++) {
                int y = virtualBounds.y + (row * rowHeight);
                for (int column = 0; column < columns; column++) {
                    int x = virtualBounds.x + (column * columnWidth);
                    Rectangle bounds = new Rectangle(x, y, columnWidth, rowHeight);
                    GrabberWorker worker = new GrabberWorker(id, this, bounds);
                    System.out.println(id);
                    id++;
                    startThread(worker);
                }
            }
        }

        @Override
        public double getScale() {
            return scale;
        }

        @Override
        public void invalidate() {
            super.invalidate();
            scale = getScaleFactorToFit(virtualBounds.getSize(), getSize());
        }

        @Override
        public Dimension getPreferredSize() {
            return new Dimension(500, 200);
        }

        @Override
        protected void paintComponent(Graphics g) {
            super.paintComponent(g);
            Graphics2D g2d = (Graphics2D) g.create();
            g2d.setColor(Color.RED);
            for (Integer id : pieces.keySet()) {
                PuzzlePiece piece = pieces.get(id);
                Rectangle bounds = piece.getBounds();
                BufferedImage img = piece.getImage();
                g2d.drawImage(img, bounds.x, bounds.y, this);
                // If you want to see each sections bounds, uncomment below...
                //g2d.draw(bounds);
            }
            g2d.dispose();
        }

        @Override
        public void setPiece(int id, PuzzlePiece piece) {
            pieces.put(id, piece);
            repaint();
        }

        protected void startThread(GrabberWorker worker) {
            Thread thread = new Thread(worker);
            thread.setDaemon(true);
            thread.start();
        }
    }

    public class PuzzlePiece {

        private final Rectangle bounds;
        private final BufferedImage img;

        public PuzzlePiece(Rectangle bounds, BufferedImage img) {
            this.bounds = bounds;
            this.img = img;
        }

        public Rectangle getBounds() {
            return bounds;
        }

        public BufferedImage getImage() {
            return img;
        }

    }

    public interface Puzzler {

        public void setPiece(int id, PuzzlePiece piece);

        public double getScale();

    }

    public class GrabberWorker implements Runnable {

        private Rectangle bounds;
        private Puzzler puzzler;
        private int id;

        private volatile PuzzlePiece parked;
        private ReentrantLock lckParked;

        public GrabberWorker(int id, Puzzler puzzler, Rectangle bounds) {
            this.id = id;
            this.bounds = bounds;
            this.puzzler = puzzler;
            lckParked = new ReentrantLock();
        }

        protected void process(PuzzlePiece piece) {
//            puzzler.setPiece(bounds, chunks.get(chunks.size() - 1));

            puzzler.setPiece(id, piece);
        }

        protected void publish(PuzzlePiece piece) {

            lckParked.lock();
            try {
                parked = piece;
            } finally {
                lckParked.unlock();
            }
            SwingUtilities.invokeLater(new Runnable() {
                @Override
                public void run() {
                    lckParked.lock();
                    try {
                        process(parked);
                    } finally {
                        lckParked.unlock();
                    }
                }
            });

        }

        @Override
        public void run() {
            try {
                Robot bot = new Robot();
                while (true) {
                    BufferedImage img = bot.createScreenCapture(bounds);

                    double scale = puzzler.getScale();
                    Rectangle scaled = new Rectangle(bounds);
                    scaled.x *= scale;
                    scaled.y *= scale;
                    scaled.width *= scale;
                    scaled.height *= scale;

                    BufferedImage imgScaled = getScaledInstance(img, scale);

                    publish(new PuzzlePiece(scaled, imgScaled));

                    Thread.sleep(500);

                }
            } catch (AWTException | InterruptedException exp) {
                exp.printStackTrace();
            }
        }

    }

    public static Rectangle getVirtualBounds() {

        Rectangle bounds = new Rectangle(0, 0, 0, 0);

        GraphicsEnvironment ge = GraphicsEnvironment.getLocalGraphicsEnvironment();
        GraphicsDevice lstGDs[] = ge.getScreenDevices();
        for (GraphicsDevice gd : lstGDs) {

            bounds.add(gd.getDefaultConfiguration().getBounds());

        }

        return bounds;

    }

    public static double getScaleFactorToFit(Dimension original, Dimension toFit) {

        double dScale = 1d;

        if (original != null && toFit != null) {

            double dScaleWidth = getScaleFactor(original.width, toFit.width);
            double dScaleHeight = getScaleFactor(original.height, toFit.height);

            dScale = Math.min(dScaleHeight, dScaleWidth);

        }

        return dScale;

    }

    public static double getScaleFactor(int iMasterSize, int iTargetSize) {

        double dScale = (double) iTargetSize / (double) iMasterSize;
        return dScale;

    }

    public static BufferedImage getScaledInstance(BufferedImage img, double dScaleFactor) {

        return getScaledInstance(img, dScaleFactor, RenderingHints.VALUE_INTERPOLATION_BILINEAR, true);

    }

    protected static BufferedImage getScaledInstance(BufferedImage img, double dScaleFactor, Object hint, boolean bHighQuality) {

        BufferedImage imgScale = img;

        int iImageWidth = (int) Math.round(img.getWidth() * dScaleFactor);
        int iImageHeight = (int) Math.round(img.getHeight() * dScaleFactor);

//        System.out.println("Scale Size = " + iImageWidth + "x" + iImageHeight);
        if (dScaleFactor <= 1.0d) {

            imgScale = getScaledDownInstance(img, iImageWidth, iImageHeight, hint, bHighQuality);

        } else {

            imgScale = getScaledUpInstance(img, iImageWidth, iImageHeight, hint, bHighQuality);

        }

        return imgScale;

    }

    protected static BufferedImage getScaledDownInstance(BufferedImage img,
            int targetWidth,
            int targetHeight,
            Object hint,
            boolean higherQuality) {

        int type = (img.getTransparency() == Transparency.OPAQUE)
                ? BufferedImage.TYPE_INT_RGB : BufferedImage.TYPE_INT_ARGB;

        BufferedImage ret = (BufferedImage) img;
        if (targetHeight > 0 || targetWidth > 0) {
            int w, h;
            if (higherQuality) {
                // Use multi-step technique: start with original size, then
                // scale down in multiple passes with drawImage()
                // until the target size is reached
                w = img.getWidth();
                h = img.getHeight();
            } else {
                // Use one-step technique: scale directly from original
                // size to target size with a single drawImage() call
                w = targetWidth;
                h = targetHeight;
            }

            do {
                if (higherQuality && w > targetWidth) {
                    w /= 2;
                    if (w < targetWidth) {
                        w = targetWidth;
                    }
                }

                if (higherQuality && h > targetHeight) {
                    h /= 2;
                    if (h < targetHeight) {
                        h = targetHeight;
                    }
                }

                BufferedImage tmp = new BufferedImage(Math.max(w, 1), Math.max(h, 1), type);
                Graphics2D g2 = tmp.createGraphics();
                g2.setRenderingHint(RenderingHints.KEY_INTERPOLATION, hint);
                g2.drawImage(ret, 0, 0, w, h, null);
                g2.dispose();

                ret = tmp;
            } while (w != targetWidth || h != targetHeight);
        } else {
            ret = new BufferedImage(1, 1, type);
        }
        return ret;
    }

    protected static BufferedImage getScaledUpInstance(BufferedImage img,
            int targetWidth,
            int targetHeight,
            Object hint,
            boolean higherQuality) {

        int type = BufferedImage.TYPE_INT_ARGB;

        BufferedImage ret = (BufferedImage) img;
        int w, h;
        if (higherQuality) {
            w = img.getWidth();
            h = img.getHeight();
        } else {
            w = targetWidth;
            h = targetHeight;
        }

        do {
            if (higherQuality && w < targetWidth) {
                w *= 2;
                if (w > targetWidth) {
                    w = targetWidth;
                }
            }

            if (higherQuality && h < targetHeight) {
                h *= 2;
                if (h > targetHeight) {
                    h = targetHeight;
                }
            }

            BufferedImage tmp = new BufferedImage(w, h, type);
            Graphics2D g2 = tmp.createGraphics();
            g2.setRenderingHint(RenderingHints.KEY_INTERPOLATION, hint);
            g2.drawImage(ret, 0, 0, w, h, null);
            g2.dispose();

            ret = tmp;
            tmp = null;
        } while (w != targetWidth || h != targetHeight);
        return ret;
    }
}

Now, if you're feeling really adventurous, you could update this idea so that if the underlying image for a section didn't change and the scale didn't change, it didn't send that to the UI, which might help reduce some of the overhead ... and no, I don't have code to do that ;)