How to call CreateWindowEx from Ruby?

Question

I have the following code which is fine if I give invalid parameters (though, obviously doesn't work), but whenever I give accurate parameters, ruby segfaults. I'm inclined to believe that this is a problem with my code and/or the ability of ruby to actually call this API function, but I'd like some more input. I've tried with both Win32API and DL::Importer with the same results. Is there any way to get this to work?

For the curious, there's full background available here, including attempts with Win32API and DL::Importer on different branches. You are looking for examples/windows-test in both cases.

EDIT: I've managed to get RegisterClassEx to work, but this still isn't helping. Ruby is silently crashing in CreateWindowEx.

The following gives output like this:

wndproc: 4293787656
hInstance: 4194304
Entering RegisterClassEx
Window Class: 49795
Entering CreateWindowEx

EDIT 2: My code in progress has grown a little large to be pasting it all into SE. If you want all the background, you can see it at the link above. I've tried to keep everything relevant included here though.

class Windows
  def initialize
    puts "wndproc: #{Win32::User32::WNDPROC}"

    hInstance = Win32::Kernel32::GetModuleHandle(DL::NULL)
    puts "hInstance: #{hInstance}"

    puts "Entering RegisterClassEx"

    @window_class_struct = Win32::User32::WNDCLASSEX.malloc
    @window_class_struct.cbSize        = Win32::User32::WNDCLASSEX.size
    @window_class_struct.style         = Win32::User32::CS_HREDRAW | Win32::User32::CS_VREDRAW
    @window_class_struct.lpfnWndProc   = Win32::User32::WNDPROC
    @window_class_struct.cbClsExtra    = 0
    @window_class_struct.cbWndExtra    = 0
    @window_class_struct.hInstance     = hInstance
    @window_class_struct.hIcon         = 0
    @window_class_struct.hCursor       = 0
    @window_class_struct.hbrBackground = Win32::User32::COLOR_WINDOWFRAME
    @window_class_struct.lpszMenuName  = DL::NULL
    @window_class_struct.lpszClassName = 'ruby-skype'
    @window_class_struct.hIconSm       = 0

    p @window_class_struct

    @window_class = Win32::User32::RegisterClassEx(@window_class_struct.to_i)
    puts "Window Class: #{@window_class}"

    puts "Entering CreateWindowEx"
    @window = Win32::User32::CreateWindowEx(0, 'ruby-skype', 'ruby-skype', Win32::User32::WS_OVERLAPPEDWINDOW,
                                    0, 0, 200, 200, DL::NULL, DL::NULL, DL::NULL)
    puts "Exited CreateWindowEx"

    p @window
  end

  module Win32

    module Types
      def included(m)
        m.module_eval {
          include ::DL::Win32Types

          # @see http://msdn.microsoft.com/en-us/library/windows/desktop/aa383751.aspx
          typealias('HBRUSH', 'HANDLE')
          typealias('HCURSOR', 'HANDLE')
          typealias('HICON', 'HANDLE')
          typealias('HMENU', 'HANDLE')
          typealias('HMODULE', 'HANDLE')
          typealias('LPCTSTR', 'unsigned char *')
          typealias('LPVOID', 'void *')
          typealias('WNDPROC', 'void *') # Actually a function pointer
          typealias('WNDCLASSEX', 'void *') # struct
        }
      end
      module_function :included
    end

    module User32
      extend DL
      extend DL::Importer
      dlload 'user32'
      include Types

      extern 'HWND CreateWindowEx(DWORD, LPCTSTR, LPCTSTR, DWORD, int, int, int, int, HWND, HMENU, HINSTANCE)'

      WNDPROC = set_callback DL::TYPE_LONG, 4 do |window_handle, message_id, wParam, lParam|
        puts "WM: #{message_id}"
      end
    end
  end
end

Windows.new

Answer 1

Solution: Use ffi. For whatever reason, it just doesn't work in DL (Win32API uses DL under the hood)

Full credit goes here, wherever this is (I can't read Japanese): http://www19.atwiki.jp/tmtbnc/m/pages/56.html

My guess is it's because DL doesn't seem to support stdcall, but I honestly don't know enough about it to know.

The FFI solution as used by me is below:

class Windows
  def initialize
    hInstance = Win32::GetModuleHandle(nil)

    @window_class = Win32::WNDCLASSEX.new
    @window_class[:style]         = Win32::CS_HREDRAW | Win32::CS_VREDRAW
    @window_class[:lpfnWndProc]   = method(:message_pump)
    @window_class[:hInstance]     = hInstance
    @window_class[:hbrBackground] = Win32::COLOR_WINDOWFRAME
    @window_class[:lpszClassName] = FFI::MemoryPointer.from_string 'ruby-skype'

    @window = Win32::CreateWindowEx(Win32::WS_EX_LEFT, ::FFI::Pointer.new(@window_class.atom), 'ruby-skype', Win32::WS_OVERLAPPEDWINDOW,
                                    0, 0, 0, 0, Win32::NULL, Win32::NULL, hInstance, nil)
  end

  def message_pump(window_handle, message_id, wParam, lParam)
    puts "WM: #{message_id}"
    Win32::DefWindowProc(window_handle, message_id, wParam, lParam)
  end

  module Win32
    extend FFI::Library
    ffi_lib('user32', 'kernel32')
    ffi_convention(:stdcall)

    private

    def self._func(*args)
      attach_function *args
      case args.size
        when 3
          module_function args[0]
        when 4
          module_function args[0]
          alias_method(args[1], args[0])
          module_function args[1]
      end
    end

    public

    ULONG_PTR = FFI::TypeDefs[:ulong]
    LONG_PTR = FFI::TypeDefs[:long]

    ULONG = FFI::TypeDefs[:ulong]
    LONG = FFI::TypeDefs[:long]
    LPVOID = FFI::TypeDefs[:pointer]
    INT = FFI::TypeDefs[:int]
    BYTE = FFI::TypeDefs[:uint16]
    DWORD = FFI::TypeDefs[:ulong]
    BOOL = FFI::TypeDefs[:int]
    UINT = FFI::TypeDefs[:uint]
    POINTER = FFI::TypeDefs[:pointer]
    VOID = FFI::TypeDefs[:void]

    HWND = HICON = HCURSOR = HBRUSH = HINSTANCE = HGDIOBJ =
        HMENU = HMODULE = HANDLE = ULONG_PTR
    LPARAM = LONG_PTR
    WPARAM = ULONG_PTR
    LPCTSTR = LPMSG = LPVOID
    LRESULT = LONG_PTR
    ATOM = BYTE
    NULL = 0

    WNDPROC = callback(:WindowProc, [HWND, UINT, WPARAM, LPARAM], LRESULT)

    class WNDCLASSEX < FFI::Struct
      layout :cbSize, UINT,
             :style, UINT,
             :lpfnWndProc, WNDPROC,
             :cbClsExtra, INT,
             :cbWndExtra, INT,
             :hInstance, HANDLE,
             :hIcon, HICON,
             :hCursor, HCURSOR,
             :hbrBackground, HBRUSH,
             :lpszMenuName, LPCTSTR,
             :lpszClassName, LPCTSTR,
             :hIconSm, HICON

      def initialize(*args)
        super
        self[:cbSize] = self.size
        @atom = 0
      end

      def register_class_ex
        (@atom = Win32::RegisterClassEx(self)) != 0 ? @atom : raise("RegisterClassEx Error")
      end

      def atom
        @atom != 0 ? @atom : register_class_ex
      end
    end # WNDCLASSEX

    class POINT < FFI::Struct
      layout :x, LONG,
             :y, LONG
    end

    class MSG < FFI::Struct
      layout :hwnd, HWND,
             :message, UINT,
             :wParam, WPARAM,
             :lParam, LPARAM,
             :time, DWORD,
             :pt, POINT
    end

    _func(:RegisterWindowMessage, :RegisterWindowMessageA, [LPCTSTR], UINT)
    _func(:GetModuleHandle, :GetModuleHandleA, [LPCTSTR], HMODULE)
    _func(:RegisterClassEx, :RegisterClassExA, [LPVOID], ATOM)
    _func(:CreateWindowEx, :CreateWindowExA, [DWORD, LPCTSTR, LPCTSTR, DWORD, INT, INT, INT, INT, HWND, HMENU, HINSTANCE, LPVOID], HWND)
    _func(:GetMessage, :GetMessageA, [LPMSG, HWND, UINT, UINT], BOOL)
    _func(:TranslateMessage, [LPVOID], BOOL)
    _func(:DispatchMessage, :DispatchMessageA, [LPVOID], BOOL)
    _func(:DefWindowProc, :DefWindowProcA, [HWND, UINT, WPARAM, LPARAM], LRESULT)

    # @!group Predefined WindowHandle's
    #
    # These are WindowHandle's provided by the Win32 API for special purposes.

    # Target for SendMessage(). Broadcast to all windows.
    HWND_BROADCAST = 0xffff
    # Used as a parent in CreateWindow(). Signifies that this should be a message-only window.
    HWND_MESSAGE = -3

    # @!endgroup

    # CreateWindow Use Default Value
    CW_USEDEFAULT = 0x80000000

    COLOR_WINDOW = 5
    COLOR_WINDOWFRAME = 6

    # @!group Class Style contants.

    CS_VREDRAW = 0x0001
    CS_HREDRAW = 0x0002

    # @!group Window Style constants
    #
    # This is only a subset.
    # @see http://msdn.microsoft.com/en-us/library/windows/desktop/ms632600.aspx

    WS_BORDER =      0x00800000
    WS_CAPTION =     0x00C00000
    WS_DISABLED =    0x08000000
    WS_OVERLAPPED =  0x00000000
    WS_POPUP =       0x80000000
    WS_SIZEBOX =     0x00040000
    WS_SYSMENU =     0x00080000
    WS_THICKFRAME =  0x00040000
    WS_MAXIMIZEBOX = 0x00010000
    WS_MINIMIZEBOX = 0x00020000
    WS_OVERLAPPEDWINDOW = WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_THICKFRAME | WS_MINIMIZEBOX | WS_MAXIMIZEBOX
    WS_POPUPWINDOW = WS_POPUP | WS_BORDER | WS_SYSMENU

    # @!group Window Extended Style constants
    #
    # This is only a subset.
    # @see http://msdn.microsoft.com/en-us/library/windows/desktop/ff700543.aspx

    WS_EX_LEFT = 0

    # @!endgroup
  end
end