Simulating String.split using StringTokenizer

Question

I'm converting code from an existing application to compile against a Java 1.1 compiler for a custom piece of hardware. This means that I can't use String.split(regex) to convert my existing string into an array.

I created a method which should give the same result as String.split(regex) but there's something wrong with it and I can't figure out what.

Code:

private static String[] split(String delim, String line) {
  StringTokenizer tokens = new StringTokenizer(line, delim, true);
  String previous = "";
  Vector v = new Vector();

  while(tokens.hasMoreTokens()) {
    String token = tokens.nextToken();

    if(!",".equals(token)) {
      v.add(token);
    } else if(",".equals(previous)) {
      v.add("");
    } else {
      previous = token;
    }
  }

  return (String[]) v.toArray(new String[v.size()]);
}

Sample input:

RM^RES,0013A2004081937F,,9060,1234FF

Sample output:

String line = "RM^RES,0013A2004081937F,,9060,1234FF";
String[] items = split(",", line);

for(String s : items) {
    System.out.println(" [ " + s + " ] ");
}

[ RM^RES ] [ 0013A2004081937F ] [ ] [ ] [ 9060 ] [ ] [ 1234FF ]

Desired output:

[ RM^RES ] [ 0013A2004081937F ] [ ] [ 9060 ] [ 1234FF ]

---

Old code that I'm trying to convert:

String line = "RM^RES,0013A2004081937F,,9060,1234FF";
String[] items = line.split(",");

for(String s : items) {
    System.out.println(" [ " + s + " ] ");
}

[ RM^RES ] [ 0013A2004081937F ] [ ] [ 9060 ] [ 1234FF ]

Answer 1

I modified the code and tested it. It works (don't forget to avoid hard-coding the "," so you can use the function for any delimiter):

private static String[] split(String delim, String line) {

    StringTokenizer tokens = new StringTokenizer(line, delim, true);
    String previous = delim;
    Vector v = new Vector();

    while (tokens.hasMoreTokens()) {
        String token = tokens.nextToken();

        if (!delim.equals(token)) {
            v.add(token);
        } else if (previous.equals(delim)) {
            v.add("");
        }
        previous = token;
    }

    return (String[]) v.toArray(new String[v.size()]);
}

Answer 2

Almost everything is right. Almost, because you forget to "clear" value of previous. Try this:

if(!",".equals(token)) {
  v.add(token);
  previous = "";
} else if(",".equals(previous)) {
  v.add("");
  previous = "";
} else {
  previous = token;
}

Answer 3

How about not using StringTokenizer at all:

private static String[] split(String delim, String line) {
    String current = line;
    int index = line.indexOf(delim);
    Vector vector = new Vector();
    while (index != -1) {
        vector.add(current.substring(0, index));
        current = current.substring(index + 1);
        index = current.indexOf(delim);
    }
    vector.add(current);

    return (String[]) vector.toArray(new String[vector.size()]);
}

Answer 4

You can try it this way

 public static void main(String[] args) throws ParseException {
    for (String s : split(",", "RM^RES,0013A2004081937F, ,9060,1234FF")) {
        System.out.print(" [ " + s + " ] ");
    }
  }

private static String[] split(String delim, String line) {
    StringTokenizer tokens = new StringTokenizer(line, delim);
    String[] v = new String[tokens.countTokens()];
    int i = 0;
    while (tokens.hasMoreTokens()) {
        v[i] = tokens.nextToken();
        i++;
    }
    return v;
}

Answer 5

I think you should not assume anything about the underlying delimiter.

    public static String[] split(String line, String delim) {
        Vector v = new Vector();
        final String EMPTY_STRING = "";
        StringTokenizer st = new StringTokenizer(line, delim, true);
        while (st.hasMoreTokens()) {
            String token = st.nextToken();

            if (token.equals(delim)) {
                if (v.isEmpty() || v.size() > 0 && !EMPTY_STRING.equals(v.get(v.size() - 1))) {
                    v.add(EMPTY_STRING);
                }
            } else {
                v.add(token);
            }
        }

        return (String[])v.toArray(new String[v.size()]);
    }