If you want to stay with pure RELAX NG (and avoid alternative solutions like embedded Schematron rules), you could use:
start = Tickmarks
Tickmarks = element tickmarks {
(
attribute from { xsd:float },
attribute to { xsd:float },
Tick-float*)
| (
attribute from { xsd:date },
attribute to { xsd:date },
Tick-date*)
}
Tick-float = element tick {
attribute value { xsd:float },
attribute label { text }
}
Tick-date = element tick {
attribute value { xsd:date },
attribute label { text }
}