Commits

Ran Jiao committed 5ffc80b

add tokenizer so that we can use it to read text files.

Comments (0)

Files changed (10)

 IF(UNIX)
   MESSAGE("Building in Linux like system")
   ADD_DEFINITIONS(-DLINUX)
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -g")
 ENDIF()
 
 IF(WIN32)

src/Demos/HelloWorld/HelloWorld.cpp

 
 void HelloWorld::InitApp()
 {
-  AppBase::InitApp();
+  NATIVE_APP::InitApp();
 
   m_effect = new CGEffect("illum.cgfx");
 }

src/Framework/App/AppBase.h

 #define __APPBASE_H__
 
 #include <string>
-
+#include "Utils/Logger.h"
 #include "../Common.h"
 
 #if defined USE_GUI_QT
     /// Called after every frame is rendered
     virtual void AfterRender(){};
 
+    /// TODO: Return arguments from command line
+    virtual STRING GetCmdArguments()
+    {
+      LOG_WARNING1("GetCmdArguments not implemented!");
+      return EMPTY_STRING;
+    };
+
     virtual void OnClose() = 0;
     virtual bool OnKey(unsigned key, bool down);;
 

src/Framework/Utils/Tokenizer.h

+#ifndef __TOKENIZER_H__
+#define __TOKENIZER_H__
+
+#include "Utils/Util.h"
+
+namespace JustRenderIt
+{
+  class DLL_DECLARE Tokenizer
+  {
+  protected:
+    char *m_buffer, *m_crt, *m_next;
+    char m_delim[32], m_comment[32];
+
+    unsigned m_length;
+  public:
+    Tokenizer()
+    {
+      strcpy(m_delim, " \t\n\r");
+      strcpy(m_comment, "#");
+      m_buffer = NULL;
+      m_length = -1;
+    };
+
+    ~Tokenizer()
+    {
+      SAFE_DELETE_ARRAY(m_buffer);
+    }
+
+    void SetDelim(char* delim)
+    {
+      unsigned length = max(size_t(31), strlen(delim));
+      memcpy(m_delim, delim, length);
+      m_delim[32] = 0;
+    }
+
+    void SetComment(char* comment)
+    {
+      unsigned length = max(size_t(31), strlen(comment));
+      memcpy(m_comment, comment, length);
+      m_comment[32] = 0;
+    }
+
+    char* GetBuffer(){ return m_buffer; };
+    unsigned GetLength(){ return m_length; };
+
+    bool LoadFromFile(STRING filename)
+    {
+      if(filename.empty())
+        return false;
+      if(!FileExists(filename))
+        return false;
+
+      FILE* infile = fopen(filename.c_str(), "rb");
+      if(!infile)
+      {
+        assert(false);
+        return false;
+      }
+
+      SAFE_DELETE(m_buffer);
+
+      unsigned length = FileLength(filename);
+      printf("file length: %d\n", length);
+      m_buffer = new char[length+1];
+      m_buffer[length] = '0';
+      fread(m_buffer, 1, length, infile);
+
+
+      m_length = length;
+      m_next = m_crt =  NULL;
+      return true;
+    }
+
+    bool LoadFromMemory(const char* data, unsigned length = 0)
+    {
+      if(!data)
+        return false;
+
+      SAFE_DELETE(m_buffer);
+      unsigned newLength = length == 0 ? strlen(data) : length;
+      m_buffer = new char[newLength+1];
+      m_length = newLength;
+
+      memcpy(m_buffer, data, newLength);
+
+      m_buffer[newLength] = 0;
+
+      m_crt = m_next = NULL;
+
+      return true;
+    }
+
+    STRING CrtToken()
+    {
+      STRING result;
+      char buffer[128]; // TOOD: avoid overflow
+      char* ptr;
+
+      assert(m_next > m_crt);
+
+      int index = 0;
+      for(ptr = m_crt ; ptr != m_next && index < 128; ptr++ && index++)
+        buffer[index] = *ptr;
+
+      buffer[index] = 0;
+      return STRING(buffer);
+    }
+
+    void Next()
+    {
+      // initialize the pointers
+      if(m_crt == m_next && m_crt == NULL)
+      {
+        m_crt = m_buffer;
+        SkipDelim(&m_crt);
+
+        m_next = m_crt;
+        SkipNoDelim(&m_next);
+        return;
+      }
+
+      assert(m_crt && m_next);
+      SkipDelim(&m_next);
+      m_crt = m_next;
+      SkipNoDelim(&m_next);
+      return;
+    }
+
+    unsigned SkipLine(char** ptr)
+    {
+      assert(ptr);
+      char* buffer = *ptr;
+      unsigned traveled = 0;
+
+      // Skip curent line
+      for( ; *buffer != 0 && *buffer !='\n'; buffer++ && traveled++) ;
+
+      // Skip \n
+      buffer++;
+      traveled++;
+
+      *ptr = buffer;
+      return traveled;
+    }
+
+    unsigned SkipNoDelim(char** ptr)
+    {
+      assert(ptr);
+
+      unsigned traveled = 0;
+      int delimCount = strlen(m_delim);
+      int commCount = strlen(m_comment);
+      char* buffer = *ptr;
+
+      // jump to next line if we hit comment at first
+      for(int i=0; i<commCount; i++)
+        if(*buffer == m_comment[i])
+        {
+          traveled += SkipLine(&buffer);
+        }
+
+      bool isNotDelim = true, isComment = false;
+      for( ; *buffer != 0 && isNotDelim && !isComment; buffer++ && traveled++)
+      {
+        isNotDelim = true;
+        for(int i=0; i<commCount; i++)
+          if(*buffer == m_comment[i])
+          {
+            isComment = true; // stop going if we meet a comment
+          }
+
+        for(int i=0; i<delimCount; i++)
+          if(*buffer == m_delim[i])
+          {
+            isNotDelim = false;
+            break;
+          }
+      }
+
+      if(traveled)
+      {
+        buffer--;
+        traveled--;
+      }
+
+      *ptr = buffer;
+      return traveled;
+    }
+
+    unsigned SkipDelim(char** ptr)
+    {
+      assert(ptr);
+
+      unsigned traveled = 0;
+      int delimCount = strlen(m_delim);
+      int commCount = strlen(m_comment);
+      char* buffer = *ptr;
+
+      // jump to next line if we hit comment at first
+      for(int i=0; i<commCount; i++)
+        if(*buffer == m_comment[i])
+        {
+          traveled += SkipLine(&buffer);
+        }
+
+      bool isDelim = true, isComment = false;
+      for( ; *buffer != 0 && isDelim && !isComment; buffer++ && traveled++)
+      {
+        isDelim = false;
+        for(int i=0; i<commCount; i++)
+          if(*buffer == m_comment[i])
+          {
+            // conntinue to next line if we meed comments
+            traveled += SkipLine(&buffer);
+          }
+
+        for(int i=0; i<delimCount; i++)
+          if(*buffer == m_delim[i])
+          {
+            isDelim = true;
+            break;
+          }
+      }
+
+      if(traveled)
+      {
+        buffer--;
+        traveled--;
+      }
+
+      *ptr = buffer;
+      return traveled;
+    }
+  };
+};
+
+#endif /* __TOKENIZER_H__ */

src/Framework/Utils/Util.cpp

 
 unsigned JustRenderIt::FileLength(STRING filename)
 {
+  if(!FileExists(filename))
+    return -1;
+
   FILE *file = fopen(filename.c_str(), "rb");
   unsigned length;
 

src/Framework/Utils/Util.h

     dest = ToBool(src);
     return false;
   }
-}
 
-#ifndef min
-#define min(a, b) ((a)>(b) ? (b) : (a))
+  template<typename T>
+  T max(T a, T b)
+  {
+    return a > b ? a : b;
+  }
+
+  template<typename T>
+  T min(T a, T b)
+  {
+    return a < b ? a : b;
+  }
+}; // EOF namespace JustRenderIt
+
 #endif
-#ifndef max
-#define max(a, b) ((a)<(b) ? (b) : (a))
-#endif
-#endif

src/Tests/test_resource.cpp

 TEST(Resource, Create)
 {
   TestResourceLoader* loader = new TestResourceLoader();
+
   Handle h1 = loader->LoadFromFile("file1");
   TestResource* r = (TestResource*)(g_resourceMgr->GetResource(h1));
+
+  delete loader;
   EXPECT_TRUE(r != NULL);
 };
 

src/Tests/test_tokenizer.cpp

+#include "Utils/Tokenizer.h"
+#include "Common.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+using namespace JustRenderIt;
+
+const char* filename = "../src/Tests/test_tokenizer.txt";
+const unsigned length = FileLength(filename);
+
+TEST(Tokenizer, LoadFromMemory)
+{
+  char in_string[] = "this is a test string = 123;";
+
+  Tokenizer t1, t2, t3;
+  t1.LoadFromMemory(in_string, strlen(in_string));
+  t2.LoadFromMemory(in_string);
+  t3.LoadFromMemory(in_string, 4);
+
+  EXPECT_TRUE(t1.GetLength() == strlen(in_string));
+  EXPECT_TRUE(strlen(t1.GetBuffer()) == t1.GetLength());
+  EXPECT_TRUE(t1.GetLength() == t2.GetLength());
+  EXPECT_TRUE(strlen(t2.GetBuffer()) == t2.GetLength());
+
+  EXPECT_TRUE(strlen(t3.GetBuffer()) == t3.GetLength());
+  EXPECT_TRUE(strcmp(t3.GetBuffer(), "this") == 0);
+}
+
+TEST(Tokenizer, LoadFromFile)
+{
+  Tokenizer t;
+  EXPECT_TRUE(t.LoadFromFile(filename));
+
+  EXPECT_EQ(length, t.GetLength());
+}
+
+TEST(Tokenizer, SkipDelim)
+{
+  char in_string[] = "    this is a test string = 123;";
+
+  Tokenizer t;
+  t.LoadFromMemory(in_string, strlen(in_string));
+
+  char* ptr = t.GetBuffer();
+  unsigned count = t.SkipDelim(&ptr);
+
+  EXPECT_TRUE( strcmp( in_string+count, ptr ) == 0 );
+}
+
+TEST(Tokenizer, SkipNoDelim)
+{
+  const char in_string[] = "this is a test string = 123;";
+
+  Tokenizer t;
+  t.LoadFromMemory(in_string, strlen(in_string));
+
+  char* ptr = t.GetBuffer();
+  unsigned count = t.SkipNoDelim(&ptr);
+  EXPECT_TRUE(count == 4);
+
+  const char in_string2[] = "    another string";
+  Tokenizer t2;
+  t2.LoadFromMemory(in_string2);
+  char* ptr2 = t2.GetBuffer();
+  unsigned count2 = t2.SkipNoDelim(&ptr2);
+  EXPECT_TRUE(count2 == 0);
+}
+
+TEST(Tokenizer, SkipLine)
+{
+  const char in_string[] = "123456\n11\n11";
+
+  Tokenizer t;
+  t.LoadFromMemory(in_string);
+
+  char* ptr = t.GetBuffer();
+  t.SkipLine(&ptr);
+  EXPECT_TRUE( strcmp(ptr, "11\n11") == 0);
+
+  t.SkipLine(&ptr);
+  EXPECT_TRUE( strcmp(ptr, "11") == 0);
+}
+
+TEST(Tokenizer, CrtToken)
+{
+  const char in_string[] = " 1this is \n exam # and comment \n other content";
+
+  Tokenizer t;
+  t.LoadFromMemory(in_string);
+
+  t.Next();
+  STRING result = t.CrtToken();
+  EXPECT_TRUE( result == "1this" );
+
+  t.Next();
+  result = t.CrtToken();
+  EXPECT_TRUE( result == "is" );
+
+  t.Next();
+  result = t.CrtToken();
+  EXPECT_TRUE( result == "exam" );
+
+  t.Next();
+  result = t.CrtToken();
+  EXPECT_TRUE( result == "other" );
+}

src/Tests/test_tokenizer.cpp~

+#include "Utils/Tokenizer.h"
+#include "Common.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+using namespace JustRenderIt;
+
+TEST(Tokenizer, Load)
+{
+  char in_string[] = "this is a test string = 123;";
+
+//  Tokenizer t;
+//  t.LoadFromMemory(in_string, strlen(in_string));
+
+}

src/Tests/test_tokenizer.txt

+a = 123;
+b = aaa;
+csfe = a31d
+df1 = 31dz