Reading lines of text from file with Using and Yield Return

Question

I have the method below which uses Yield Return to read large ( >1m ) lines of text from a file.

    private static IEnumerable<string> ReadLineFromFile(TextReader fileReader)
    {
        using (fileReader)
        {
            string currentLine;
            while ((currentLine = fileReader.ReadLine()) != null)
            {
                yield return currentLine;
            }
        }
    }

I need to be able to write every 10 lines returned from this method to different files.

How do I consume this method without enumerating all the lines?

Any answer is very much appreciated.

Answer 1

I think I finally got it working :-)

        var listOfBufferedLines = ReadLineFromFile(ReadFilePath);

        var listOfLinesInBatch = new List<string>();
        foreach (var line in listOfBufferedLines)
        {
            listOfLinesInBatch.Add(line);

            if (listOfLinesInBatch.Count % 1000 == 0)
            {
                Console.WriteLine("Writing Batch.");
                WriteLinesToFile(listOfLinesInBatch, LoadFilePath);
                listOfLinesInBatch.Clear();
            }
        }

        // writing the remaining lines
        WriteLinesToFile(listOfLinesInBatch, LoadFilePath);

Answer 2

If you run the below code, you can see all you need to do is call your method within a foreach loop, and it will iterate it one at a time, you just need to buffer it somewhere to a batch size of your choice.

static void Main (string [] args)
{
    int batch_size = 5;
    string buffer = "";
    foreach (var c in EnumerateString("THISISALONGSTRING")) 
    {               
        // Check if it's time to split the batch
        if (buffer.Length >= batch_size) 
        {
            // Process the batch
            buffer = ProcessBuffer(buffer);
        }

        // Add to the buffer
        buffer += c;
    }

    // Process the remaining items
    ProcessBuffer(buffer);

    Console.ReadLine();
}

public static string ProcessBuffer(string buffer)
{
    Console.WriteLine(buffer);  
    return "";
}

public static IEnumerable<char> EnumerateString(string huh)
{
    for (int i = 0; i < huh.Length; i++) {
        Console.WriteLine("yielded: " + huh[i]);
        yield return huh[i];
    }
}

Answer 3

Definitely not an elegant way to solve this, but it will work

static void Main(string[] args)
        {

            try
            {
                System.IO.TextReader readFile = new StreamReader(@"C:\Temp\test.txt");
                int count = 0;
                List<string> lines= new List<string>();
                foreach (string line in ReadLineFromFile(readFile))
                {
                    if (count == 10)
                    {
                        count = 0;
                        ProcessChunk(lines);
                        lines.Add(line);
                    }
                    else
                    {
                        lines.Add(line);
                        count++;
                    }

                }
                //PROCESS the LINES
                ProcessChunk(lines);

                Console.ReadKey();
            }
            catch (IOException ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }

        private static void ProcessChunk(List<string> lines)
        {
            Console.WriteLine("----------------");
            lines.ForEach(l => Console.WriteLine(l));
            lines.clear();
        }

        private static IEnumerable<string> ReadLineFromFile(TextReader fileReader)
        {
            using (fileReader)
            {
                string currentLine;
                while ((currentLine = fileReader.ReadLine()) != null)
                {
                    yield return currentLine;
                }
            }
        }