Set
EnableClientScript="false"
in RequiredFieldValidator control. It will help.
<asp:Panel runat="server" DefaultButton="secondButton">
<table>
<tr>
<td>
<asp:TextBox runat="server" ID="validateMe"></asp:TextBox></td>
<td>
<asp:RequiredFieldValidator ID="firstValidator" runat="server" ErrorMessage="First check not valid" Text="*" ControlToValidate="validateMe" EnableClientScript="false" ValidationGroup="firstGroup"></asp:RequiredFieldValidator>
</td>
<td>
<asp:RequiredFieldValidator ID="secondValidator" runat="server" ErrorMessage="Second check not valid" Text="*" ControlToValidate="validateMe" EnableClientScript="false" ValidationGroup="secondGroup"></asp:RequiredFieldValidator>
</td>
</tr>
<tr>
<td>
<asp:Button runat="server" ID="firstButton" Text="V1" ValidationGroup="firstGroup" /></td>
<td>
<asp:Button runat="server" ID="secondButton" Text="V2" ValidationGroup="secondGroup" /></td>
</tr>
</table>
<asp:ValidationSummary ID="firstSummary" runat="server" ValidationGroup="firstGroup" />
<asp:ValidationSummary ID="secondSummary" runat="server" ValidationGroup="secondGroup" />
</asp:Panel>