Creating objects from IDataReader

Question

I have created the following to create User objects with an ObservableCollection of UserModule objects called UserModules.

I have approx. 100000 user records, each user could have up to 10 module records and this is taking minutes to finish.

This may mean changing from IDataReader, open to suggestion. Can someone suggest a more efficient way to do this?

public void LoadUsers()
{
    clsDAL.SQLDBAccess db = new clsDAL.SQLDBAccess("USERS");
    clsDAL.SQLDBAccess db_user_modules = new clsDAL.SQLDBAccess("USERS");
    try
    {
        db.setCommandText(@"SELECT * FROM Users");

        using (var reader = db.ExecuteReader())
        {
            while (reader.Read())
            {
                var user = new User();
                MapUser(reader, user);
                _users.Add(user);

                db_user_modules.setCommandText(@"SELECT MODULE_ID, USER_MODULE_ACCESS FROM USER_MODULE_SECURITY Where USER_ID = " + user.User_ID);

                using (var reader_user_modules = db_user_modules.ExecuteReader())
                {
                    while (reader_user_modules.Read())
                    {
                        MapUserModule(reader_user_modules, user);
                    }
                }
            }
        }
    }
    catch(Exception ex)
    {
        MessageBox.Show(ex.Message);
        throw;
    }
    finally
    {
        db = null;
    }

}

MapUser

public static void MapUser(IDataRecord record, User user)
{
    try
    {
        user.User_ID = NullSafeGetter.GetValueOrDefault<int>(record, "USER_ID");
        user.Username = NullSafeGetter.GetValueOrDefault<string>(record, "USERNAME");
        user.Name = NullSafeGetter.GetValueOrDefault<string>(record, "NAME");
        user.Job_Title = NullSafeGetter.GetValueOrDefault<string>(record, "JOB_TITLE");
        user.Department = NullSafeGetter.GetValueOrDefault<string>(record, "DEPARTMENT");
        user.Company = NullSafeGetter.GetValueOrDefault<string>(record, "COMPANY");
        user.Phone_Office = NullSafeGetter.GetValueOrDefault<string>(record, "PHONE_OFFICE");
        user.Phone_Mobile = NullSafeGetter.GetValueOrDefault<string>(record, "PHONE_MOBILE");
        user.Email = NullSafeGetter.GetValueOrDefault<string>(record, "EMAIL");
        user.Password = NullSafeGetter.GetValueOrDefault<string>(record, "PASSWORD");
        user.Last_Login = NullSafeGetter.GetValueOrDefault<DateTime>(record, "LAST_LOGIN");
        user.Status = NullSafeGetter.GetValueOrDefault<int>(record, "STATUS");
        user.Session_Timeout = NullSafeGetter.GetValueOrDefault<int>(record, "SESSION_TIMEOUT");
    }
    catch (Exception ex)
    {
        MessageBox.Show("Mapping User error: " + ex.Message);
        throw;
    }
}

MapUserModule

private static void MapUserModule(IDataRecord record, User user)
{
    try
    {
        int m_id = NullSafeGetter.GetValueOrDefault<int>(record, "MODULE_ID");
        int uma = NullSafeGetter.GetValueOrDefault<int>(record, "USER_MODULE_ACCESS");

        user.UserModules.Add(new Users.UserModule(user.User_ID, m_id, uma));
    }
    catch (Exception ex)
    {
        throw new Exception("Mapping UserModule error:\n" + ex.Message);
    }
}

Answer 1

public IEnumerable<UserModule> GetUserModules()
{
    using(var db = ....)
    db.setCommandText("SELECT * FROM USERMODULES");
    using (var reader = db.ExecuteReader())
    {
        while (reader.Read())
        {
            var userId = reader[...];
            var m_id = reader[...];
            var uma = reader[...];
            yield return new UserModule (userid, m_id, uma)
        }
    }
}

public IEnumerable<User> GetUsers()
{
    var userModulesLookup = GetUserModules().ToLookup(x => x.UserId);
    using (var db = ...)
    {
        db.setCommandText("SELECT * FROM USERS");
        using (var reader = db.ExecuteReader())
        {
            while (reader.Read())
            {
                var userId = reader["userId"];
                ...blah blah blah...
                var user = return new User();
                user.Modules = new ObservableCollection<UserModule>
                                     (userModulesLookup[userId]);
                ...blah blah blah...
                yield return user;
            }
        }
    }
}

public void LoadUsers()
{
    var users = GetUsers();
    foreach(var u in users)
        _users.Add(u);
}

Answer 2

As far as I know, there is no faster solution than using a DataReader.

I would recommend you profile the code to see what is taking up most of the time. IIRC, adding a large number of items to an observable collection one at a time is slow. Try adding them to a List<> temporarily to try and isolate the problem.