How to create an RavenDb Index that gets the latest version of a category?

Question

Related to this question: RavenDB Get By List of IDs? What I have is not working:

public class Entity
{
    public int CategoryId; //Unique identifier for the category the price applies to.
    public int Price; //Changes with time
    public DateTime Timestamp; //DateTime.UtcNow
}
public class LastEntityIndex : AbstractIndexCreationTask<Entity, Entity>
{
    public LastEntityIndex()
    {
        Map = prices => prices.Select(a => new { a.CategoryId, a.Price, a.Timestamp });

        Reduce = prices => from price in prices
                           group price by price.CategoryId
                                 into g
                                 let a = g.OrderByDescending(a => a.Timestamp).FirstOrDefault()
                                 select new
                                 {
                                     a.CategoryId,
                                     a.Price,
                                     a.Timestamp
                                 };
    }
}
public class LastEntity
{
    public int CategoryId;
    public DateTime Timestamp;
    public Entity LastEntity;
}
public class LastReportIndex : AbstractIndexCreationTask<Entity, LastEntity>
{
    public LastReportIndex()
    {
        Map = reports => reports.Select(a => new { a.CategoryId, a.Timestamp, LastEntity = a });

        Reduce = reports => from report in reports
                            group report by report.CategoryId
                                into g
                                let a = g.OrderByDescending(a => a.Timestamp).FirstOrDefault()
                                select new
                                {
                                    a.CategoryId,
                                    a.Timestamp,
                                    a.LastEntity
                                };
    }
}

I'd like to create an index to get the latest record by each category. But none of the above are working. Any help is hugely appreciated. Going away from sql for this new project and evaluating Raven. So far it seems very powerful and spot on for what we need, but it's hard with the paradigm shift away from sql dbs.

Thank you very much.

PS Using this to retrieve the records:

public List<Entity> GetLastEntityForCategoryIds(List<int> categoryIds)
    {
        using (var session = _documentStore.OpenSession())
        {
            return session.Query<LastEntity, LastReportIndex>()
                                    .Where(x => x.CategoryId.In(categoryIds))
                                    .Select(a => a.LastEntity)
                                    .ToList();
        }
    }

Obviously the 'LastEntityIndex' is not something that I'd use long term (it's just to try out see if that works), because the real Entity has a lot more fields that just 3 and copying them all and maintaining it will be very hard.

Answer 1

You are correct that you have to select the entire document, not part of the document. By default, a query returns whole documents.

If you want to return only part of a document, you have to use projections, or transformers, or do like you did and get the whole thing and then pull out the parts you want.

However, you're doing a lot of stuff that I don't think is necessary. For example, you are ordering in the reduce and taking a single item. That's not a great use of an index. You should probably just have a simple map and do the ordering by your query.

See this GIST and read my comments. A few other things are commented in there as well.

And cheers to you as well!

Answer 2

Right, I figured out what was going on. If I do .Select on Mapped/Reduced query, the defaults are returned. If I do .ToList().Select or Single() etc I get the correct result.

Here is a unit test I've set up to demonstrate what I am talking about.

using Raven.Client.Indexes;
using Raven.Tests.Helpers;
using System;
using System.Linq;
using Xunit; //XUnit
//using Microsoft.VisualStudio.TestTools.UnitTesting; //MSTest

namespace RavenDBTest.Tests
{
    //[TestClass] //MSTest
    public class RavenIndexTest : RavenTestBase
    {
        //[TestMethod] //MSTest
        [Fact] //XUnit
        public void CanIndexAndQuery()
        {
            var timestamp = DateTime.Now;
            var report1 = new Entity { CategoryId = 123, Price = 51, Timestamp = timestamp.AddSeconds(-20) };
            var report2 = new Entity { CategoryId = 123, Price = 62, Timestamp = timestamp.AddSeconds(-10) };
            var report3 = new Entity { CategoryId = 123, Price = 73, Timestamp = timestamp };

            using (var store = NewDocumentStore())
            {
                new LatestEntity_Index().Execute(store);

                using (var session = store.OpenSession())
                {
                    session.Store(report1);
                    session.Store(report2);
                    session.Store(report3);

                    session.SaveChanges();
                }

                //WILL PASS
                using (var session = store.OpenSession())
                {
                    var result = session.Query<LastEntity, LatestEntity_Index>()
                        .Customize(customization => customization.WaitForNonStaleResultsAsOfNow())
                        .FirstOrDefault()
                        .LastReport;

                    AssertLatestResult(timestamp, result);
                }

                //WILL FAIL
                using (var session = store.OpenSession())
                {
                    var result = session.Query<LastEntity, LatestEntity_Index>()
                        .Customize(customization => customization.WaitForNonStaleResultsAsOfNow())
                        .Select(a => a.LastReport)
                        .FirstOrDefault();

                    AssertLatestResult(timestamp, result);
                }
            }
        }

        private static void AssertLatestResult(DateTime timestamp, Entity result)
        {
            //MSTest:
            //Assert.AreEqual(123, result.CategoryId, "Category Does Not Match");
            //Assert.AreEqual(73, result.Price, "Latest Price Does Not Match");
            //Assert.AreEqual(timestamp, result.Timestamp, "Latest Timestamp Does Not Match");

            //XUnit:
            Assert.Equal(123, result.CategoryId);
            Assert.Equal(73, result.Price);
            Assert.Equal(timestamp, result.Timestamp);
        }
    }

    public class Entity
    {
        public int CategoryId {get;set;} //Unique identifier for the category the price applies to.
        public int Price {get;set;} //Changes with time
        public DateTime Timestamp {get;set;} //DateTime.UtcNow
    }

    public class LastEntity
    {
        public int CategoryId { get; set; }
        public DateTime Timestamp { get; set; }
        public Entity LastReport { get; set; }
    }

    public class LatestEntity_Index : AbstractIndexCreationTask<Entity, LastEntity>
    {
        public LatestEntity_Index()
        {
            Map = reports => from report in reports
                             select new LastEntity
                             {
                                 CategoryId = report.CategoryId,
                                 Timestamp = report.Timestamp,
                                 LastReport = report,
                             };

            Reduce = reports => from report in reports
                                group report by report.CategoryId
                                    into g
                                    let last = g.OrderBy(a=>a.Timestamp).Last()
                                    select new LastEntity
                                    {
                                        CategoryId = g.Key,
                                        Timestamp = last.Timestamp,
                                        LastReport = last.LastReport,
                                    };
        }
    }
}

I hope this helps someone.

@Matt Johnson, how efficient/crappy are these indexes? Performance wise? and Space wise? Cheers for your help by the way. :)