Based on the Firefox source code I've developed the following code that solves my problem:
HMENU hMenu = GetSystemMenu(hWnd, FALSE);
if (hMenu)
{
MENUITEMINFO mii;
mii.cbSize = sizeof(MENUITEMINFO);
mii.fMask = MIIM_STATE;
mii.fType = 0;
// update the options
mii.fState = MF_ENABLED;
SetMenuItemInfo(hMenu, SC_RESTORE, FALSE, &mii);
SetMenuItemInfo(hMenu, SC_SIZE, FALSE, &mii);
SetMenuItemInfo(hMenu, SC_MOVE, FALSE, &mii);
SetMenuItemInfo(hMenu, SC_MAXIMIZE, FALSE, &mii);
SetMenuItemInfo(hMenu, SC_MINIMIZE, FALSE, &mii);
mii.fState = MF_GRAYED;
WINDOWPLACEMENT wp;
GetWindowPlacement(hWnd, &wp);
switch (wp.showCmd)
{
case SW_SHOWMAXIMIZED:
SetMenuItemInfo(hMenu, SC_SIZE, FALSE, &mii);
SetMenuItemInfo(hMenu, SC_MOVE, FALSE, &mii);
SetMenuItemInfo(hMenu, SC_MAXIMIZE, FALSE, &mii);
SetMenuDefaultItem(hMenu, SC_CLOSE, FALSE);
break;
case SW_SHOWMINIMIZED:
SetMenuItemInfo(hMenu, SC_MINIMIZE, FALSE, &mii);
SetMenuDefaultItem(hMenu, SC_RESTORE, FALSE);
break;
case SW_SHOWNORMAL:
SetMenuItemInfo(hMenu, SC_RESTORE, FALSE, &mii);
SetMenuDefaultItem(hMenu, SC_CLOSE, FALSE);
break;
}
LPARAM cmd = TrackPopupMenu(hMenu, (TPM_RIGHTBUTTON | TPM_NONOTIFY | TPM_RETURNCMD),
GET_X_LPARAM(lParam), GET_Y_LPARAM(lParam), NULL, hWnd, NULL);
if (cmd) PostMessage(hWnd, WM_SYSCOMMAND, cmd, 0);
}
Apparently you need to manually set the enabled and disabled items.