move tokenizer.h to include/avian/util/string.h, merge in the String declaration from object-writer/tools.h

include/avian/tools/object-writer/tools.h

@@ -12,6 +12,9 @@
 #define AVIAN_TOOLS_H_
 
 #include <stdlib.h>
+
+#include <avian/util/string.h>
+
 #include "environment.h"
 
 namespace avian {
@@ -38,24 +41,12 @@ public:
   virtual void write(uint8_t byte);
 };
 
-class String {
-public:
-  const char* text;
-  size_t length;
-
-  String(const char* text);
-  
-  inline String(const char* text, size_t length):
-    text(text),
-    length(length) {}
-};
-
 class SymbolInfo {
 public:
   unsigned addr;
-  String name;
+  util::String name;
 
-  inline SymbolInfo(uint64_t addr, const String& name):
+  inline SymbolInfo(uint64_t addr, const util::String& name):
     addr(addr),
     name(name) {}
 
@@ -78,7 +69,7 @@ public:
 
 class StringTable : public Buffer {
 public:
-  unsigned add(String str);
+  unsigned add(util::String str);
 };
 
 template<class T>

include/avian/util/string.h

@@ -8,27 +8,37 @@
    There is NO WARRANTY for this software.  See license.txt for
    details. */
 
-#ifndef TOKENIZER_H
-#define TOKENIZER_H
+#ifndef AVIAN_UTIL_STRING_H
+#define AVIAN_UTIL_STRING_H
 
-namespace vm {
+#include <string.h>
+
+namespace avian {
+namespace util {
+
+class String {
+public:
+  const char* text;
+  size_t length;
+
+  String(const char* text):
+    text(text),
+    length(strlen(text)) {}
+  
+  inline String(const char* text, size_t length):
+    text(text),
+    length(length) {}
+};
 
 class Tokenizer {
  public:
-  class Token {
-   public:
-    Token(const char* s, unsigned length): s(s), length(length) { }
-
-    const char* s;
-    unsigned length;
-  };
 
   Tokenizer(const char* s, char delimiter):
     s(s), limit(0), delimiter(delimiter)
   { }
 
-  Tokenizer(const char* s, unsigned length, char delimiter):
-    s(s), limit(s + length), delimiter(delimiter)
+  Tokenizer(String str, char delimiter):
+    s(str.text), limit(str.text + str.length), delimiter(delimiter)
   { }
 
   bool hasMore() {
@@ -36,10 +46,10 @@ class Tokenizer {
     return s != limit and *s != 0;
   }
 
-  Token next() {
+  String next() {
     const char* p = s;
     while (s != limit and *s and *s != delimiter) ++s;
-    return Token(p, s - p);
+    return String(p, s - p);
   }
 
   const char* s;
@@ -47,6 +57,7 @@ class Tokenizer {
   char delimiter;
 };
 
-} // namespace
+} // namespace util
+} // namespace avain
 
-#endif//TOKENIZER_H
+#endif//AVIAN_UTIL_STRING_H

makefile

@@ -939,7 +939,11 @@ generated-code = \
 	$(build)/type-name-initializations.cpp \
 	$(build)/type-maps.cpp
 
-vm-depends := $(generated-code) $(wildcard $(src)/*.h) $(wildcard $(src)/codegen/*.h) $(wildcard $(src)/codegen/compiler/*.h)
+vm-depends := $(generated-code) \
+	$(wildcard $(src)/*.h) \
+	$(wildcard $(src)/codegen/*.h) \
+	$(wildcard $(src)/codegen/compiler/*.h) \
+	$(shell find include -name '*.h')
 
 vm-sources = \
 	$(src)/vm/system/$(system).cpp \

src/classpath-common.h

@@ -11,10 +11,11 @@
 #ifndef CLASSPATH_COMMON_H
 #define CLASSPATH_COMMON_H
 
-#include "tokenizer.h"
-
+#include <avian/util/string.h>
 #include <avian/util/runtime-array.h>
 
+using namespace avian::util;
+
 namespace vm {
 
 object
@@ -217,13 +218,13 @@ loadLibrary(Thread* t, const char* path, const char* name, bool mapName,
   for (Tokenizer tokenizer(path, t->m->system->pathSeparator());
        tokenizer.hasMore();)
   {
-    Tokenizer::Token token(tokenizer.next());
+    String token(tokenizer.next());
 
     unsigned fullNameLength = token.length + 1 + nameLength;
     THREAD_RUNTIME_ARRAY(t, char, fullName, fullNameLength + 1);
 
     snprintf(RUNTIME_ARRAY_BODY(fullName), fullNameLength + 1,
-             "%.*s/%s", token.length, token.s, name);
+             "%.*s/%s", token.length, token.text, name);
 
     lib = loadLibrary(t, RUNTIME_ARRAY_BODY(fullName));
     if (lib) break;

src/finder.cpp

@@ -8,15 +8,17 @@
    There is NO WARRANTY for this software.  See license.txt for
    details. */
 
-#include "zlib-custom.h"
 #include <avian/vm/system/system.h>
-#include "tokenizer.h"
+#include <avian/util/string.h>
+#include <avian/util/runtime-array.h>
+
+#include "zlib-custom.h"
 #include "finder.h"
 #include "lzma.h"
 
-#include <avian/util/runtime-array.h>
 
 using namespace vm;
+using namespace avian::util;
 
 namespace {
 
@@ -658,12 +660,12 @@ addTokens(System* s, Element** first, Element** last, Allocator* allocator,
           const char* jarName, unsigned jarNameBase, const char* tokens,
           unsigned tokensLength, const char* bootLibrary)
 {
-  for (Tokenizer t(tokens, tokensLength, ' '); t.hasMore();) {
-    Tokenizer::Token token(t.next());
+  for (Tokenizer t(String(tokens, tokensLength), ' '); t.hasMore();) {
+    String token(t.next());
 
     RUNTIME_ARRAY(char, n, jarNameBase + token.length + 1);
     memcpy(RUNTIME_ARRAY_BODY(n), jarName, jarNameBase);
-    memcpy(RUNTIME_ARRAY_BODY(n) + jarNameBase, token.s, token.length);
+    memcpy(RUNTIME_ARRAY_BODY(n) + jarNameBase, token.text, token.length);
     RUNTIME_ARRAY_BODY(n)[jarNameBase + token.length] = 0;
           
     add(s, first, last, allocator, RUNTIME_ARRAY_BODY(n),
@@ -813,9 +815,9 @@ parsePath(System* s, Allocator* allocator, const char* path,
   Element* first = 0;
   Element* last = 0;
   for (Tokenizer t(path, s->pathSeparator()); t.hasMore();) {
-    Tokenizer::Token token(t.next());
+    String token(t.next());
 
-    add(s, &first, &last, allocator, token.s, token.length, bootLibrary);
+    add(s, &first, &last, allocator, token.text, token.length, bootLibrary);
   }
 
   return first;

src/tools/object-writer/elf.cpp

@@ -72,6 +72,7 @@
 namespace {
 
 using namespace avian::tools;
+using namespace avian::util;
 
 template<class AddrTy>
 struct ElfTypes {

src/tools/object-writer/mach-o.cpp

@@ -43,6 +43,7 @@
 namespace {
 
 using namespace avian::tools;
+using namespace avian::util;
 
 typedef int cpu_type_t;
 typedef int cpu_subtype_t;

src/tools/object-writer/pe.cpp

@@ -86,6 +86,7 @@ pad(unsigned n)
 }
 
 using namespace avian::tools;
+using namespace avian::util;
 
 template<unsigned BytesPerWord, PlatformInfo::Architecture Architecture>
 class WindowsPlatform : public Platform {

src/tools/object-writer/tools.cpp

@@ -15,14 +15,12 @@
 
 #include <avian/tools/object-writer/tools.h>
 
+using namespace avian::util;
+
 namespace avian {
 
 namespace tools {
 
-String::String(const char* text):
-  text(text),
-  length(strlen(text)) {}
-
 Buffer::Buffer():
   capacity(100),
   length(0),