preCalculate file stream checksum

Question

i'm trying to ensure an output File integrity in case of disk out of space , network problem ,or any anyException that might occur during the streaming to file process . is there a way to precalculate the FileStream checkSum before writing to disk then check if the file was written properly. it sounds a bit nonsensical for me , that a system validates the integrity of its own exported XML through checkSum , normaly it's the job of the other end to verify if the the consumed file lives up to the file produced by the other system .

but it's a requirement i have to implement.

her's the stream i write as a file :

 String xmlTransfer ="";
    File testFile =  new File("testFile.xml");
    InputStream in = new ByteArrayInputStream(xmlTransfer.getBytes("utf-8"));
    FileOutputStream out = new FileOutputStream(testFile)
     byte[] buffer = new byte[2048];
               int bytesRead;
               while ((bytesRead = in.read(buffer)) != -1) {
                   out.write(buffer, 0, bytesRead);
               }
               out.close();
               in.close();

Answer 1

try this :

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.security.MessageDigest;


public class CheckSumFileTest 
{

    private File buildChecksumFile(File fileToCheck, String filePrefix, String checksumAlgorithm) throws Exception
    {
        String checksum = null;
        File checksumFile = null;
        String tempDir = System.getProperty("java.io.tmpdir");
        try {
            checksumFile = new File(tempDir, filePrefix+"."+ checksumAlgorithm.toLowerCase());
            checksumFile.createNewFile();
            checksumFile.deleteOnExit();
        } catch (Exception e1) {
            e1.printStackTrace();
            throw e1;
        }
        FileWriter fw = null;
        try {
            checksum = checkSum(fileToCheck,checksumAlgorithm);
            fw = new FileWriter(checksumFile);
            fw.write(checksum);
        } catch (Exception e) {
            e.printStackTrace();
            throw e;
        }
        finally
        {
            if(fw !=null)
                fw.close();
        }

        return checksumFile;
    }

    private static String checkSum(File file, String checksumAlgorithm) throws Exception
    {
        MessageDigest digest = MessageDigest.getInstance(checksumAlgorithm);

        InputStream input = null;
        StringBuffer sb = new StringBuffer();
        try{
            input = new FileInputStream(file);
            byte[] buffer = new byte[8192];
            do {
                int read = input.read(buffer);
                if(read <= 0)
                    break;
                digest.update(buffer, 0, read);
            } while(true);
            byte[] sum = digest.digest();

            for (int i = 0; i < sum.length; i++) {
                sb.append(Integer.toString((sum[i] & 0xff) + 0x100, 16).substring(1));
            }

        }catch(IOException io)
        {

        }finally{
            if(input != null)
                input.close();
        }

        return sb.toString();
    }

    private static String checkSumInStream(InputStream stream, String checksumAlgorithm) throws Exception
    {
        MessageDigest digest = MessageDigest.getInstance(checksumAlgorithm);

        InputStream input = null;
        StringBuffer sb = new StringBuffer();
        try{
            input = stream;
            byte[] buffer = new byte[8192];
            do {
                int read = input.read(buffer);
                if(read <= 0)
                    break;
                digest.update(buffer, 0, read);
            } while(true);
            byte[] sum = digest.digest();

            for (int i = 0; i < sum.length; i++) {
                sb.append(Integer.toString((sum[i] & 0xff) + 0x100, 16).substring(1));
            }

        }catch(IOException io)
        {

        }finally{
            if(input != null)
                input.close();
        }

        return sb.toString();
    }

    private boolean checkIntegrity(String targetFileName, String checksumFileName, String checksumAlgorithm) throws Exception
    {
        FileInputStream stream = null;
        BufferedReader br = null;
        InputStreamReader ipsr = null;
        File checksumFile = null;
        String checksumString="";
        File targetFile = new File(targetFileName);
        try{
            checksumFile = new File(checksumFileName);
            stream = new FileInputStream(checksumFile);
            ipsr = new InputStreamReader(stream);
            br = new BufferedReader(ipsr);

            //In checksum file : only one line to read
            checksumString = br.readLine();

        }finally
        {
            if(br != null)
                br.close();
            if(ipsr != null)
                ipsr.close();
            if(stream != null)
                stream.close();
        }


        if(checksumString.equals(checkSum(targetFile,checksumAlgorithm)))
        {
            return true;
        }
        else
        {
            return false;
        }
    }



    /**
     * @param args
     */
    public static void main(String[] args) 
    {
        String str = "Amine";
        InputStream stream = new ByteArrayInputStream(str.getBytes());
        //step1
        try {
            System.out.println(checkSumInStream(stream,"MD5"));
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        //step2
        File file = new File("c:/test.txt");

        // if file doesnt exists, then create it
        if (!file.exists()) {
            try {
                file.createNewFile();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }

        FileWriter fw;
        BufferedWriter bw;

        try {
            fw = new FileWriter(file.getAbsoluteFile());
            bw = new BufferedWriter(fw);
            bw.write(str);
            bw.close();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

        try {
            System.out.println(checkSum(file, "MD5"));
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        System.out.println("Done");
    }

}

Answer 2

No, you can't figure out how much data will come from a stream in advance. That's simply not how streams are meant to work.

What you could do, if you are writing both ends of the code, is to first calculate the file size on the sending end and send that before sending the file contents itself.

Answer 3

The best way is to catch exception. If something go wrong an exception will be launched and you could remove the partially written file in this case.

A second way is to have a in-memory stream before writing down to the filesystem but it consumes memory.

A third way is to ensure the destination disk capacity (new File(path).getFreeSpace())

The MD5 check sounds too slow for me in regards of the question.

Answer 4

You should check by MD5, not file size

You can calculate your MD5 while you're reading the stream.

See https://stackoverflow.com/a/304350/3230038

Then, after saving the file, you can generate the md5 again and compare

UPDATE - here's my more detailed idea for this. I am assuming that you just want to calculate the MD5 without having to bring the whole byte[] into memory. In this case, I think you have 2 options

1. calculate MD5 on the fly, as you're saving, then after saving, check md5 again (if you're on linux you can just use md5sum)
2. calculate MD5 in a first pass, then save the file in a second pass.

for example

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.NullOutputStream;


public class MD5OnTheFly {

    /**
     * @param args
     * @throws NoSuchAlgorithmException 
     * @throws IOException 
     */
    public static void main(String[] args) throws NoSuchAlgorithmException, IOException {

        long ini = System.currentTimeMillis();

        File file = new File("/home/leoks/Downloads/VirtualBox-4.3.0.tar");

        System.out.println("size:"+file.length());

        InputStream is = new FileInputStream(file);

        MessageDigest md = MessageDigest.getInstance("MD5");

        DigestInputStream dis = new DigestInputStream(is, md);

        IOUtils.copy(dis, new NullOutputStream());

        byte[] digest = md.digest();

        StringBuffer hexString = new StringBuffer();
        for (int i = 0; i < digest.length; i++) {
            String hex = Integer.toHexString(0xff & digest[i]);
            if (hex.length() == 1)
                hexString.append('0');
            hexString.append(hex);
        }

        System.out.println(hexString);

        long end = System.currentTimeMillis();
        System.out.println(end-ini+" millis");


    }
}

returns

410859520
dda81aea75a83b1489662c6bcd0677e4
1413 millis

and then

[leoks@home ~]$ md5sum /home/leoks/Downloads/VirtualBox-4.3.0.tar
dda81aea75a83b1489662c6bcd0677e4  /home/leoks/Downloads/VirtualBox-4.3.0.tar
[leoks@home ~]$