App stops at foreach number 3286090

Question

Basically I have a problem where the application just stops after loop 3286090 or batch 1735.

I have a list of 1894 of validated addresses at this point in the application and now makes all possible combinations and calculates the distance and travel time per batch. This sub function calls to a local web service that takes 60 to 180 seconds to complete for each batch and writes the result to a .csv file. (Writing this to an excel file with existing excel libraries convulates the memory excessively so it wasn't an option.)

There is no exception. There is no system log. And every single "break on exception" option with ctrl+alt+e is enabled.

if (startNumber <= batchnumber)
{
    calculateRouteInfo(waypointDescArrayList, batchnumber, address);
}

Code seems to fail here. The moment batchnumber reaches 1735 and compares it with the startNumber (which in this case I have tried entering 1734 to redo the last batch / 1735 to do the current batch and try to skip it at 1736 or higher.)

No matter what number above 1736 the application just reaches the end at that specific number of comparing X with batchnumber 1735 even when I just tell the application to compare it with a higher number like 1800. It just ends there.

I tried fiddling around with conditions and checking for memory leaks but this is not the case. Also the web service functions normally on any other batches before that number. Manually making that batch and sending it to the web-service functions normally as well.

Here is the full code.

private static void preprocessCalculation(xLocate.AddressResponse[] foundAddressess)
{
    int batchnumber = 1;
    List<xRoute.WaypointDesc[]> waypointDescArrayList = new List<xRoute.WaypointDesc[]>();
    foreach (var foundAddress in foundAddressess)
    {// 1. foundAddresses containts 1894 foundAddress' 
        foreach (var address in foundAddress.wrappedResultList)
        {// 2. Each foundAddress.wrappedResultlist containts (in this case) 1x address.
            GC.Collect();
            GC.WaitForPendingFinalizers();
            GC.Collect();
            var watch = Stopwatch.StartNew();
            foreach (var foundAddressDest in foundAddressess)
            {// 3. Here we combine each address with every address and for each address we make up a batch of combinations that we calculate.
                foreach (var addressDest in foundAddressDest.wrappedResultList)
                { // 4.
                    #region Add waypointDesc
                    var waypointDescList = new List<xRoute.WaypointDesc>();
                    waypointDescList.Add(new xRoute.WaypointDesc()
                    {
                        linkType = xRoute.LinkType.AUTO_LINKING,
                        wrappedCoords = new xRoute.Point[] { 
                            new xRoute.Point() { 
                                point = new xRoute.PlainPoint() {
                                    x = address.coordinates.point.x, 
                                    y = address.coordinates.point.y 
                                }
                            }
                        }
                    });
                    waypointDescList.Add(new xRoute.WaypointDesc()
                    {
                        linkType = xRoute.LinkType.AUTO_LINKING,
                        wrappedCoords = new xRoute.Point[] { 
                            new xRoute.Point() { 
                                point = new xRoute.PlainPoint() {
                                    x = addressDest.coordinates.point.x, 
                                    y = addressDest.coordinates.point.y 
                                }
                            }
                        }
                    });
                    waypointDescArrayList.Add(waypointDescList.ToArray());
                    #endregion
                }
            }
            if (startNumber <= batchnumber)
            {  // This calculates the data and does not fail.
                calculateRouteInfo(waypointDescArrayList, batchnumber, address);
            }
            waypointDescArrayList.Clear();
            watch.Stop();
            elapsedtime += watch.Elapsed.TotalSeconds;
            Console.Clear();
            Console.WriteLine("Voortgang calculatie... {0}/{1} ({2}s (+{3}s))", batchnumber, totalbatches, elapsedtime, watch.Elapsed.TotalSeconds);
            batchnumber++;
        }
    }
} // 5. When going through the 1735th iteration it skips to the end of this function.

private static void calculateRouteInfo(List<xRoute.WaypointDesc[]> finalList, int batchnumber, xLocate.ResultAddress address)
{
    string startlocation = string.Format("{0}-{1}-{2}-{3}", address.country, address.postCode, address.city, address.street);
    var matrixDistance = matrixTemplate.Copy();
    matrixDistance.Rows.Add(startlocation);
    var matrixTime = matrixTemplate.Copy();
    matrixTime.Rows.Add(startlocation);

    var bulkRouteInfo = xRouteClient.calculateBulkRouteInfo(finalList.ToArray(), null, null, null);
    finalList.Clear();

    var column = 1;
    foreach (var RouteInfo in bulkRouteInfo.wrappedBulkRouteInfoResult)
    {
        matrixDistance.Rows[0][column] = RouteInfo.routeInfo.distance;
        matrixTime.Rows[0][column] = RouteInfo.routeInfo.time;
        column++;
    }
    writeOutputMatrix(fileName, batchnumber, matrixDistance, matrixTime);
}

private static void writeOutputMatrix(string fileName, int batchnumber, DataTable matrixDistance, DataTable matrixTime)
{
    string newPath = string.Format("C:/result/{0}/", fileName);
    if (!Directory.Exists(newPath))
    {
        var newDirectory = Directory.CreateDirectory(newPath);
        Console.WriteLine("Result mappen aangemaakt.");
    }

    var matrixDistanceBytes = Encoding.GetEncoding("iso-8859-1").GetBytes(matrixDistance.ToCSV());
    using (Stream s = File.Create(string.Format("{0}{1}-distance_{2}.csv", newPath, fileName, batchnumber), matrixDistanceBytes.Length))
    {
        s.Write(matrixDistanceBytes, 0, matrixDistanceBytes.Length);
        Console.WriteLine(string.Format("Result {0}{1}-distance_{2}.csv is aangemaakt!", newPath, fileName, batchnumber));
    }

    var matrixTimeBytes = Encoding.GetEncoding("iso-8859-1").GetBytes(matrixTime.ToCSV());
    using (Stream s = File.Create(string.Format("{0}{1}-time_{2}.csv", newPath, fileName, batchnumber), matrixTimeBytes.Length))
    {
        s.Write(matrixTimeBytes, 0, matrixTimeBytes.Length);
        Console.WriteLine(string.Format("Result {0}{1}-time_{2}.csv is aangemaakt!", newPath, fileName, batchnumber));
    }
}

Edit: This is my workaround.

    private static void preprocessCalculation(xLocate.AddressResponse[] foundAddressess)
    {
        int batchnumber = 1;
        List<xRoute.WaypointDesc[]> waypointDescArrayList = new List<xRoute.WaypointDesc[]>();
        foreach (var foundAddress in foundAddressess)
        {
            if (startNumber < batchnumber)
            {
                foreach (var address in foundAddress.wrappedResultList)
                {

                    GC.Collect();
                    GC.WaitForPendingFinalizers();
                    GC.Collect();
                    var watch = Stopwatch.StartNew();
                    foreach (var foundAddressDest in foundAddressess)
                    {
                        foreach (var addressDest in foundAddressDest.wrappedResultList)
                        {
                            #region Add waypointDesc
                            var waypointDescList = new List<xRoute.WaypointDesc>();
                            waypointDescList.Add(new xRoute.WaypointDesc()
                            {
                                linkType = xRoute.LinkType.AUTO_LINKING,
                                wrappedCoords = new xRoute.Point[] { 
                                new xRoute.Point() { 
                                    point = new xRoute.PlainPoint() {
                                        x = address.coordinates.point.x, 
                                        y = address.coordinates.point.y 
                                    }
                                }
                            }
                            });
                            waypointDescList.Add(new xRoute.WaypointDesc()
                            {
                                linkType = xRoute.LinkType.AUTO_LINKING,
                                wrappedCoords = new xRoute.Point[] { 
                                new xRoute.Point() { 
                                    point = new xRoute.PlainPoint() {
                                        x = addressDest.coordinates.point.x, 
                                        y = addressDest.coordinates.point.y 
                                    }
                                }
                            }
                            });
                            waypointDescArrayList.Add(waypointDescList.ToArray());
                            #endregion
                        }
                    }

                    calculateRouteInfo(waypointDescArrayList, batchnumber, address);

                    waypointDescArrayList.Clear();
                    watch.Stop();
                    elapsedtime += watch.Elapsed.TotalSeconds;
                    Console.Clear();
                    Console.WriteLine("Voortgang calculatie... {0}/{1} ({2}s (+{3}s))", batchnumber, totalbatches, elapsedtime, watch.Elapsed.TotalSeconds);
                    batchnumber++;
                }
            }
            else
            {
                batchnumber++;//debug
            }
        }
    }

I moved the condition one stack higher and it seems to solve the problem so far. But I would still like to solve this issue to why it stops mid-foreach. foundAddress.wrappedResultList can contain more than one result (In this stress test it does not.).

Answer 1

To temporarily solve the issue I have optimized the code so that it would prevent excessive and needless foreach-loops.

Then keep a count up until where the number where the crash occurs and then start a new loop continuing where you left off.

A dirty workaround but it works while I do not yet know of another solution.