You could use a scala.util.matching.Regex.MatchIterator to build something like this
type Token = String
trait TokenIterator extends Iterator[Token] {
def next: Token
def hasNext: Boolean
def pos: Int
}
class Tokenizer(regexStr: String, input: String) {
val regex = regexStr.r
def iterator: TokenIterator = new TokenIterator {
val iter = regex.findAllIn(input)
var pos = 0
def next = {
val n = iter.next
pos = iter.start
n
}
def hasNext = iter.hasNext
}
}
val str = "3 + 4 - 5"
val iter = new Tokenizer("""d+|\S+?""", str).iterator
while(iter.hasNext) {
val token = iter.next
val pos = iter.pos
println(pos + ": " + token)
}