For those who experience this problem after lots of testing and MSDN (link) I have come to conclusion that maximum single field size capable of being read by SqlDataReader
in normal reading mode is around 70MB on x64 machine, after this it needs to switch its SqlCommand
to CommandBehavior.SequentialAccess
and stream the field contents.
Example code that would work like that:
...
behaviour = CommandBehavior.SequentialAccess;
using (IDataReader reader = cmd.ExecuteReader(behaviour))
{
filler.FillData(reader, destination);
}
When you read data in a loop you need to fetch columns in order and when you reach BLOB column you should call something like this (depending on data types):
...
private string GetBlobDataString(IDataReader openedDataReader, int columnIndex)
{
StringBuilder data = new StringBuilder(20000);
char[] buffer = new char[1000];
long startIndex = 0;
long dataReceivedCount = openedDataReader.GetChars(columnIndex, startIndex, buffer, 0, 1000);
data.Append(buffer, 0, (int)dataReceivedCount);
while (dataReceivedCount == 1000)
{
startIndex += 1000;
dataReceivedCount = openedDataReader.GetChars(columnIndex, startIndex, buffer, 0, 1000);
data.Append(buffer, 0, (int)dataReceivedCount);
}
return data.ToString();
}
private byte[] GetBlobDataBinary(IDataReader openedDataReader, int columnIndex)
{
MemoryStream data = new MemoryStream(20000);
BinaryWriter dataWriter = new BinaryWriter(data);
byte[] buffer = new byte[1000];
long startIndex = 0;
long dataReceivedCount = openedDataReader.GetBytes(columnIndex, startIndex, buffer, 0, 1000);
dataWriter.Write(buffer, 0, (int)dataReceivedCount);
while (dataReceivedCount == 1000)
{
startIndex += 1000;
dataReceivedCount = openedDataReader.GetBytes(columnIndex, startIndex, buffer, 0, 1000);
dataWriter.Write(buffer, 0, (int)dataReceivedCount);
}
data.Position = 0;
return data.ToArray();
}
This should work for data up to around 1GB-1.5GB, afterwards it will break on single object not being able to reserve continuous memory block of enough size so either then flush directly to disk from buffer or split data to multiple smaller objects.