To simplify the situation, let us consider a stylesheet with text output:
XSLT 1.0
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:output method="text" encoding="UTF-8"/>
<xsl:template match="/xml">
<TOC>
<xsl:apply-templates select="chapter" mode="TOC"/>
</TOC>
</xsl:template>
<xsl:template match="chapter" mode="TOC">
<xsl:call-template name="indent">
<xsl:with-param name="amount" select="count(ancestor::chapter)"/>
</xsl:call-template>
<xsl:number format="1. " level="multiple"/>
<xsl:value-of select="long-name" />
<xsl:text> </xsl:text>
<xsl:apply-templates select="chapter" mode="TOC" />
</xsl:template>
<xsl:template name="indent">
<xsl:param name="amount"/>
<xsl:param name="char" select="'	'"/>
<xsl:if test="$amount">
<xsl:value-of select="$char"/>
<!-- recursive call -->
<xsl:call-template name="indent">
<xsl:with-param name="amount" select="$amount - 1" />
</xsl:call-template>
</xsl:if>
</xsl:template>
</xsl:stylesheet>
Applied to a (corrected)input of:
<?xml version="1.0" encoding="UTF-8"?>
<xml>
<chapter>
<long-name>Chapter 1</long-name>
<chapter>
<long-name>Chapter A</long-name>
<chapter>
<long-name>Chapter B</long-name>
</chapter>
</chapter>
<chapter>
<long-name>Chapter C</long-name>
</chapter>
</chapter>
<chapter>
<long-name>Chapter 2</long-name>
<chapter>
<long-name>Chapter D</long-name>
</chapter>
<chapter>
<long-name>Chapter E</long-name>
</chapter>
</chapter>
</xml>
produces the following result:
1. Chapter 1
1.1. Chapter A
1.1.1. Chapter B
1.2. Chapter C
2. Chapter 2
2.1. Chapter D
2.2. Chapter E