How about this way?
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes"/>
<xsl:key name="row-by-group" match="Row" use="@group" />
<xsl:template match="/">
<html>
<xsl:for-each select="Rows/Row[count(. | key('row-by-group', @group)[1]) = 1]">
<table>
<xsl:variable name="rows-in-group" select="key('row-by-group', @group)" />
<xsl:variable name="size" select="ceiling(count($rows-in-group) div 3)" />
<xsl:for-each select="$rows-in-group[position() mod $size = 1]">
<xsl:variable name="start" select="count(preceding-sibling::Row)" />
<td>
<ul class="{@group}">
<xsl:for-each select="$rows-in-group[$start <= count(preceding-sibling::Row) and count(preceding-sibling::Row) < $start + $size]">
<li><xsl:value-of select="@text"/></li>
</xsl:for-each>
</ul>
</td>
</xsl:for-each>
</table>
</xsl:for-each>
</html>
</xsl:template>
</xsl:stylesheet>
When applied to the following input:
<Rows>
<Row text='text1' group='group1'/>
<Row text='text2' group='group1'/>
<Row text='text3' group='group1'/>
<Row text='text4' group='group1'/>
<Row text='text5' group='group1'/>
<Row text='text6' group='group1'/>
<Row text='text7' group='group1'/>
<Row text='text8' group='group1'/>
<Row text='text9' group='group1'/>
<Row text='text10' group='group2'/>
<Row text='text11' group='group2'/>
<Row text='text12' group='group2'/>
<Row text='text13' group='group2'/>
<Row text='text14' group='group2'/>
<Row text='text15' group='group2'/>
<Row text='text16' group='group2'/>
<Row text='text17' group='group2'/>
<Row text='text18' group='group2'/>
<Row text='text19' group='group2'/>
<Row text='text20' group='group2'/>
<Row text='text21' group='group2'/>
<Row text='text22' group='group2'/>
<Row text='text23' group='group2'/>
</Rows>
The result is:
<?xml version="1.0" encoding="UTF-8"?>
<html>
<table>
<td>
<ul class="group1">
<li>text1</li>
<li>text2</li>
<li>text3</li>
</ul>
</td>
<td>
<ul class="group1">
<li>text4</li>
<li>text5</li>
<li>text6</li>
</ul>
</td>
<td>
<ul class="group1">
<li>text7</li>
<li>text8</li>
<li>text9</li>
</ul>
</td>
</table>
<table>
<td>
<ul class="group2">
<li>text10</li>
<li>text11</li>
<li>text12</li>
<li>text13</li>
<li>text14</li>
</ul>
</td>
<td>
<ul class="group2">
<li>text15</li>
<li>text16</li>
<li>text17</li>
<li>text18</li>
<li>text19</li>
</ul>
</td>
<td>
<ul class="group2">
<li>text20</li>
<li>text21</li>
<li>text22</li>
<li>text23</li>
</ul>
</td>
</table>
</html>
Note the root element added to the output.