Based on Radim Köhler's answer, here is what I did to fix the problem:
Created a wrapper class to allow me to register the convert function and avoid the dialect exception described in my question:
public class MyDialect : MsSql2008Dialect
{
public MyDialect()
{
RegisterFunction("ConvertDate",
new SQLFunctionTemplate(NHibernateUtil.Class,
"convert(datetime, ?1, 103)"));
}
}
Created a custom projection class to allow me call the new function as a projection in its own right:
public static class MyProjections
{
public static IProjection ConvertDate(params IProjection[] projections)
{
return Projections.SqlFunction("ConvertDate",
NHibernateUtil.DateTime, projections);
}
}
Then I check the data type that the string in column represents, then call the ConvertDate method if it is a date
if (propTypes[orderByColumn] == typeof(DateTime))
{
criteria.AddOrder(orderIsDesc
? Order.Desc(MyProjections.ConvertDate(
Projections.Property(propNames[orderByColumn])))
: Order.Asc(MyProjections.ConvertDate(
Projections.Property(propNames[orderByColumn]))));
}