I found a solution to this. The reason FindWindowEx didn't work was because it only works on child windows that have WS_CHILD style, and apparently dialog windows do not have this style. It is why EnumChildWindows won't work either (I've tried).
So the ugly solution is EnumWindows combined with GetParent to compare the handle and the text.
struct SearchData
{
public string WindowText;
public IntPtr ParentHandle;
public IntPtr ResultHandle;
}
delegate bool EnumWindowsCallback(IntPtr currentWindowHandle, ref SearchData searchData);
[DllImport("user32.dll")] static extern bool EnumWindows(EnumWindowsCallback callback, ref SearchData searchData);
[DllImport("user32.dll")] static extern IntPtr GetParent(IntPtr childHandle);
[DllImport("user32.dll")] static extern void GetWindowText(IntPtr handle, StringBuilder resultWindowText, int maxTextCapacity);
static bool Callback(IntPtr currentWindowHandle, ref SearchData searchData)
{
bool continueEnumeration = true;
IntPtr currentWindowParentHandle = GetParent(currentWindowHandle);
if (currentWindowParentHandle == searchData.ParentHandle)
{
var windowText = new StringBuilder(1024);
GetWindowText(currentWindowHandle, windowText, windowText.Capacity);
if (windowText.ToString() == searchData.WindowText)
{
searchData.ResultHandle = currentWindowHandle;
continueEnumeration = false;
}
}
return continueEnumeration;
}
IntPtr GetChildWindowHandle(string windowText, IntPtr parentHandle)
{
var searchData = new SearchData{ParentHandle=parentHandle, WindowText=windowText};
EnumWindows(Callback, ref searchData);
return searchData.ResultHandle;
}