I don't know if anyone else has figured this out, but here was my solution:
Sitefinity 5.3 IIS8 IIS Rewriting Module
Within your web.config, add the following to the end of the system.webServer node:
<rewrite>
<rules>
<!-- Redirect your page group to the base -->
<rule name="Page Group Redirect" enabled="true">
<match url="^sitefinitypagegroupname/(.*)" ignoreCase="true" />
<action type="Redirect" url="/{R:1}" />
</rule>
<!-- Rewrite your sub domain to the page group -->
<rule name="Page Group Rewrite" enabled="true">
<match url="(.*)" ignoreCase="true" />
<conditions logicalGrouping="MatchAll">
<add input="{REQUEST_FILENAME}" matchType="IsFile" negate="true" />
<add input="{REQUEST_FILENAME}" matchType="IsDirectory" negate="true" />
<add input="{URL}" negate="true" pattern="\.axd$" />
<add input="{URL}" negate="true" pattern="^/sitefinity(.*)" />
<add input="{HTTP_HOST}" pattern="^(www\.)?subdomain\.domain\.com$" ignoreCase="true" />
</conditions>
<action type="Rewrite" url="sitefinitypagegroupname/{R:1}" />
</rule>
</rules>
</rewrite>