You can add a QMdiArea
in your QMainWindow
and add any dialog as a subwindow. You can make subwindow maximized and frameless if you want.
QMdiSubWindow* w = ui->mdiArea->addSubWindow(new QColorDialog());
w->setWindowFlags(Qt::FramelessWindowHint);
w->showMaximized();