Best way to create a lot of small files?

Question

I want to create a lot (1 million) small files as quickly as possible, that's what I'm doing now:

for(long i = 0; i < veryVeryLong; i++){
    using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.None)) {
        byte[] bytes = GetFileContent(i); // no matter
        fs.Write(bytes, 0, bytes.Length);
    }
}

Can I speed up?

UPDATE

roomaroo right, I need to use Parallel, but I combined it with my function and this has a much better result. Code:

Parallel.For(0, veryVeryLogn, (i) => {
    using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.None)) {
        byte[] bytes = GetFileContent(i); // no matter
        fs.Write(bytes, 0, bytes.Length);
    }
});

Answer 1

As ChrisBint suggested, use a parallel loop.

I created three methods to write the files (code below). One uses your code above, one uses File.WriteAllBytes(...) - both these use a traditional for loop.

The third implementation used a parallel for loop.

Here are the times to create 1000 files:

FileStream: 2658ms

File.WriteAllBytes: 2555ms

Parallel.For: 617ms

So the parallel loop is four times faster than the slowest implementation. Obviously this will vary on different hardware, and your results will depend very much on your CPU and disk.

Here's the code:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Diagnostics;
using System.Threading.Tasks;

namespace FileCreator
{
    class Program
    {
        static void Main(string[] args)
        {
            string folder = @"d:\temp";

            Clean(folder);
            CreateWithParallelFileWriteBytes(folder);

            Clean(folder);
            CreateWithFileStream(folder);

            Clean(folder);
            CreateWithFileWriteBytes(folder);
        }

        private static void Clean(string folder)
        {
            if (Directory.Exists(folder))
            {
                Directory.Delete(folder, true);
            }

            Directory.CreateDirectory(folder);
        }

        private static byte[] GetFileContent(int i)
        {
            Random r = new Random(i);
            byte[] buffer = new byte[1024];
            r.NextBytes(buffer);
            return buffer;
        }

        private static void CreateWithFileStream(string folder)
        {
            var sw = new Stopwatch();
            sw.Start();

            for (int i = 0; i < 1000; i++)
            {
                string path = Path.Combine(folder, string.Format("file{0}.dat", i));

                using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.None))
                {
                    byte[] bytes = GetFileContent(i);
                    fs.Write(bytes, 0, bytes.Length);
                }
            }

            Console.WriteLine("Time for CreateWithFileStream: {0}ms", sw.ElapsedMilliseconds);
        }

        private static void CreateWithFileWriteBytes(string folder)
        {
            var sw = new Stopwatch();
            sw.Start();

            for (int i = 0; i < 1000; i++)
            {
                string path = Path.Combine(folder, string.Format("file{0}.dat", i));
                File.WriteAllBytes(path, GetFileContent(i));
            }

            Console.WriteLine("Time for CreateWithFileWriteBytes: {0}ms", sw.ElapsedMilliseconds);
        }

        private static void CreateWithParallelFileWriteBytes(string folder)
        {
            var sw = new Stopwatch();
            sw.Start();

            Parallel.For(0, 1000, (i) =>
            {
                string path = Path.Combine(folder, string.Format("file{0}.dat", i));
                File.WriteAllBytes(path, GetFileContent(i));
            });

            Console.WriteLine("Time for CreateWithParallelFileWriteBytes: {0}ms", sw.ElapsedMilliseconds);
        }
    }
}