Instead of ICaptureGraphBuilder::RenderStream, I ended up using IGraphBuilder::Connect, but only for checking if a connection is possible. After connecting the filters, the following function immediately disconnects the filters, and uses the HRESULT to determine if the connection was succesful:
Private Function Check3(graph As IGraphBuilder, filterOut As IBaseFilter, filterIn As IBaseFilter, type As AMMediaType) As Boolean
Dim result As Boolean
' Enumerate output pins
Dim enumPinsOut As IEnumPins = Nothing
If filterOut.EnumPins(enumPinsOut) = 0 Then
Dim pinOut(0) As IPin
While enumPinsOut.Next(1, pinOut, Nothing) = 0
' Enumerate output media types
Dim enumMediaTypes As IEnumMediaTypes = Nothing
If pinOut(0).EnumMediaTypes(enumMediaTypes) = 0 Then
Dim mediaType(0) As AMMediaType
While enumMediaTypes.Next(1, mediaType, Nothing) = 0
' Compare media types
If type Is Nothing OrElse
(type.majorType = Nothing OrElse type.majorType = mediaType(0).majorType) AndAlso
(type.subType = Nothing OrElse type.subType = mediaType(0).subType) AndAlso
(type.formatType = Nothing OrElse type.formatType = mediaType(0).formatType) Then
' Enumerate input pins
Dim enumPinsIn As IEnumPins = Nothing
If filterIn.EnumPins(enumPinsIn) = 0 Then
Dim pinIn(0) As IPin
While enumPinsIn.Next(1, pinIn, Nothing) = 0
' Evaluate connection return value
Dim hr As Integer = graph.Connect(pinOut(0), pinIn(0))
graph.Disconnect(pinOut(0))
result = (hr = 0)
If result = False Then Console.WriteLine(DsError.GetErrorText(hr))
Marshal.ReleaseComObject(pinIn(0))
If result = True Then Exit While
End While
End If
End If
DsUtils.FreeAMMediaType(mediaType(0))
If result = True Then Exit While
End While
End If
Marshal.ReleaseComObject(pinOut(0))
If result = True Then Exit While
End While
End If
Return result
End Function