String::tokenize()

diff --git a/doc/string.xml b/doc/string.xml
index e4ceabc..57d1060 100644 (file)
--- a/doc/string.xml
+++ b/doc/string.xml
@@ -248,6 +248,16 @@ numbers.split(" ");   // "3.14" "" ""  "42"
 </example>
 This also happens with separators at the start and the end of the string.
 </item>
+<item tag="SuperString tokenize(String &amp;delimiters)">
+The <strong>tokenize()</strong> method breaks a string into a sequence of zero or more nonempty tokens.
+The <emph>delimiters</emph> argument specifies a set of bytes that delimit the tokens in the parsed string.
+A sequence of two or more contiguous delimiter bytes in the tokenized string is
+considered to be a single delimiter, and delimiter bytes at the start or end of the string are ignored.
+<example>
+String example("aaa...bbb");
+example.tokenize(".");   // "aaa" "bbb"
+</example>
+</item>
 </description>
 </section>
 

diff --git a/src/String.h b/src/String.h
index d83c0a0..b419b68 100644 (file)
--- a/src/String.h
+++ b/src/String.h
@@ -316,6 +316,7 @@ public:
    friend bool operator != (const regex &r, const String &s);
 
    SuperString split(const String &separator);
+   SuperString tokenize(const String &delimiters);
 
 };
    /*

diff --git a/src/string.cpp b/src/string.cpp
index 22d35af..f1a662d 100644 (file)
--- a/src/string.cpp
+++ b/src/string.cpp
@@ -726,3 +726,28 @@ SuperString String::split(const String &separator)
 
    return splitted;
 }
+
+SuperString String::tokenize(const String &delim)
+{
+   SuperString tokens;
+   char        *next_token, *save;
+
+   // The string to tokenize is modified, so we need to make a copy
+
+   char *str  = new char[strlen(p->s) + 1];
+   strcpy (str, p->s);
+
+   next_token = strtok_r(str, delim.p->s, &save);
+   while (next_token != 0)
+   {
+      char *tok = new char[strlen(next_token) + 1];
+      strcpy(tok, next_token);
+
+      tokens += String(tok);
+      next_token = strtok_r(NULL, delim.p->s, &save);
+   }
+
+   delete [] str;
+
+   return tokens;
+}

diff --git a/test/string_split_join.cpp b/test/string_split_join.cpp
index fbe7946..8dc12a3 100644 (file)
--- a/test/string_split_join.cpp
+++ b/test/string_split_join.cpp
@@ -84,6 +84,15 @@ int main()
    assert(ss5[2] == "bbb");
    assert(ss5[3] == "");
 
+   //  Test tokenize
+   String s6("aaa...bbb");
+   SuperString ss6 = s6.tokenize("."); 
+   std::cout << "The string \"" << s6 << "\" tokenized on \".\":\n";
+   print_ss(ss6);
+   std::cout << "\n";
+   assert(~ss6 == 2);
+   assert(ss6[0] == "aaa");
+   assert(ss6[1] == "bbb");
 
    return 0;
 }

diff --git a/test/string_split_join.exp b/test/string_split_join.exp
index 5765ad7..d682fa0 100644 (file)
--- a/test/string_split_join.exp
+++ b/test/string_split_join.exp
@@ -9,4 +9,6 @@ The string "3.14   42" split on " ":
 ["3.14","","","42"]
 The string "-aaa-bbb-" split on "-":
 ["","aaa","bbb",""]
+The string "aaa...bbb" tokenized on ".":
+["aaa","bbb"]
 PASS string_split_join (exit status: 0)