So I was able to figure it out using a custom string key group that would let me choose what string I want to use as a Key. I'm posting it below in case anyone else wants it:
Imports System.Collections.Generic
Imports System.Collections.ObjectModel
Imports System.Globalization
Imports System.Linq
Public Class StringKeyGroup(Of T)
Inherits ObservableCollection(Of T)
Public Delegate Function GetKeyDelegate(item As T) As String
Public Property Key() As String
Get
Return m_Key
End Get
Private Set(value As String)
m_Key = value
End Set
End Property
Private m_Key As String
Public Sub New(key__1 As String)
Key = key__1
End Sub
Public Shared Function CreateGroups(items As IEnumerable(Of T), ci As CultureInfo, getKey As GetKeyDelegate, sort As Boolean) As ObservableCollection(Of StringKeyGroup(Of T))
Dim list = New ObservableCollection(Of StringKeyGroup(Of T))()
For Each itemstring In items
Dim index = -1
For i As Integer = 0 To list.Count - 1
If list(i).Key.Equals(getKey(itemstring)) Then
index = i
Exit For
End If
Next
If index = -1 Then
list.Add(New StringKeyGroup(Of T)(getKey(itemstring)))
index = list.Count - 1
End If
If index >= 0 AndAlso index < list.Count Then
list(index).Add(itemstring)
End If
Next
If sort Then
For Each group In list
group.ToList().Sort(Function(c0, c1) ci.CompareInfo.Compare(getKey(c0), getKey(c1)))
Next
End If
Return list
End Function
End Class