vadimtsushko avatar vadimtsushko committed a8aade4

UTF8 Encoding added, both for keys and string type values, for writing and reading. Example russian.dart demonstrate that feature.

Comments (0)

Files changed (15)

examples/blog.dart

 #import("dart:builtin");
 
 main(){
-  setVerboseState();
   Db db = new Db("mongo-dart-blog");
   print("Connecting to ${db.serverConfig.host}:${db.serverConfig.port}");
   DbCollection collection;

examples/russian.dart

+#import("../lib/mongo.dart");
+#import("../lib/bson/bson.dart");
+#import("dart:builtin");
+
+main(){
+  Db db = new Db("mongo-dart-test");
+  DbCollection collection;
+  print("Connecting to ${db.serverConfig.host}:${db.serverConfig.port}");
+  db.open().chain((c){  
+    collection = db.collection('test-utf8');
+    collection.remove();  
+    collection.insert({
+      'Имя': 'Вадим', 
+      'Фамилия':'Цушко',
+      'Профессия': 'Брадобрей',
+      'Шаблон': new BsonRegexp('^.adim\$')
+    });    
+    return collection.findOne();
+  }).chain((v){
+    print("Utf8 encoding demonstration. I18 strings may be used not only as values but also as keys");
+    print(v);    
+    return collection.findOne(query().eq('Имя', 'Вадим'));
+  }).chain((v){
+    print("Filtered by query().eq(): $v");
+    return collection.findOne(query().match('Имя', '^..ДИМ\$',caseInsensitive:true));    
+  }).then((v){    
+    print("Filtered by query().match(): $v");
+    db.close();  
+  });  
+}  

lib/bson/binary.dart

     while (bytes[offset++]!= 0){
        stringBytes.add(bytes[offset-1]);
     }
-    return new String.fromCharCodes(stringBytes);
+    return decodeUtf8(stringBytes);
   }
-  writeCString(List<int> charCodes){
-    bytes.setRange(offset,charCodes.length,charCodes);
-    offset += charCodes.length;
+  writeCString(String val){
+    final utfData = encodeUtf8(val);
+    bytes.setRange(offset,utfData.length,utfData);
+    offset += utfData.length;
     writeByte(0);    
  }
 

lib/bson/bson_map.dart

 class BsonMap extends BsonObject{
   Map data;
+  Map utfKeys;
   int _dataSize;
   int dataSize(){
     if (_dataSize === null){
       _dataSize = 0;
-      data.forEach((var key, var value)
+      data.forEach((String key, var value)
         {
            _dataSize += elementSize(key, value);
         });

lib/bson/bson_regexp.dart

 class BsonRegexp extends BsonObject{
   String pattern;
   String options;
+  BsonCString bsonPattern;
+  BsonCString bsonOptions;
+
   bool multiLine;
   bool caseInsensitive;
   bool verbose;
   bool dotAll;
-  bool unicodeMode;
-  bool localeDependent;
-  BsonRegexp(this.pattern,[this.multiLine,this.caseInsensitive,this.verbose,this.dotAll,this.unicodeMode,this.localeDependent]):options="";
+  bool extended;
+  BsonRegexp(this.pattern,[this.multiLine=false,this.caseInsensitive=false,this.dotAll=false,this.extended=false]):options=""{
+    createOptionsString();    
+    bsonPattern = new BsonCString(pattern,false);
+    bsonOptions = new BsonCString(options,false);
+  }
+  
   get value()=>this;
   int get typeByte() => BSON.BSON_DATA_REGEXP;  
-  byteLength()=>pattern.length+1+options.length+1;
+  byteLength()=>bsonPattern.byteLength()+bsonOptions.byteLength();
   unpackValue(Binary buffer){
     pattern = buffer.readCString();
     options = buffer.readCString();     
   }   
   createOptionsString(){
-    options = "";  
+    var buffer = new StringBuffer();
+    if (caseInsensitive === true){
+      buffer.add("i");
+    }
+    if (multiLine === true){
+      buffer.add("m");
+    }    
+    if (dotAll === true){
+      buffer.add("s");
+    }    
+    if (extended === true){
+      buffer.add("x");
+    }    
+    options = buffer.toString();
   }
   toString()=>"BsonRegexp('$pattern',options:'$options')";
-  packValue(Binary buffer){
-     createOptionsString();
-     buffer.writeCString(pattern.charCodes());
-     buffer.writeCString(options.charCodes());     
+  packValue(Binary buffer){     
+     bsonPattern.packValue(buffer);
+     bsonOptions.packValue(buffer);
   }  
 }

lib/bson/bson_string.dart

 class BsonString extends BsonObject{
   String data;
+  List<int> _utfData;  
+  List<int> get utfData(){
+    if (_utfData === null){
+      _utfData = encodeUtf8(data);
+    }
+    return _utfData;
+  }  
   BsonString(this.data);
   get value()=>data;
-  byteLength()=>data.length+1+4;
+  byteLength()=>utfData.length+1+4;
   int get typeByte() => BSON.BSON_DATA_STRING;  
   packValue(Binary buffer){
-     buffer.writeInt(data.length+1);
-     buffer.bytes.setRange(buffer.offset,data.length,data.charCodes());
-     buffer.offset += data.length;
+     buffer.writeInt(utfData.length+1);
+     buffer.bytes.setRange(buffer.offset,utfData.length,utfData);
+     buffer.offset += utfData.length;
      buffer.writeByte(0);
   }
   unpackValue(Binary buffer){
      int size = buffer.readInt32()-1; 
-     List<int> utf8Bytes = buffer.bytes.getRange(buffer.offset,size);
-     data = decodeUtf8(utf8Bytes);
+     data = decodeUtf8(buffer.bytes,buffer.offset,size);
      buffer.offset += size+1;
   }
-
 }
 class BsonCode extends BsonString{
   get value()=>this;
   BsonCode(String data):super(data);
   String toString()=>"BsonCode('$data')";  
 }
-class BsonCString extends BsonObject{
-  String data;
-  BsonCString(this.data);
-  get value()=>data;
-  byteLength()=>data.length+1;
+class BsonCString extends BsonString{
+  bool useKeyCash;
+  int get typeByte(){
+   throw "Function typeByte of BsonCString must not be called";
+  }   
+  BsonCString(String data, [this.useKeyCash = true]): super(data);
+  List<int> get utfData(){
+    if (useKeyCash){
+      return Statics.getKeyUtf8(data);
+    }
+    else {
+      return super.utfData;
+    }    
+  }  
+
+  byteLength()=>utfData.length+1;
   packValue(Binary buffer){
-     buffer.bytes.setRange(buffer.offset,data.length,data.charCodes());
-     buffer.offset += data.length;
-     buffer.writeByte(0);    
+     buffer.bytes.setRange(buffer.offset,utfData.length,utfData);
+     buffer.offset += utfData.length;
+     buffer.writeByte(0);
   }  
 }

lib/bson/bson_type.dart

     packValue(buffer);
   } 
   packValue(var buffer){ throw const Exception("must be implemented");}
-
   _ElementPair unpackElement(buffer){
     _ElementPair result = new _ElementPair();
     result.name = buffer.readCString();    
   unpackValue(var buffer){ throw const Exception("must be implemented");}
   get value()=>null;
 }
-int elementSize(name, value) {
+int elementSize(String name, value) {
   int size = 1;
   if (name !== null){
-    size += name.length + 1;
+    size += Statics.getKeyUtf8(name).length + 1;
   } 
   size += bsonObjectFrom(value).byteLength();
   return size;

lib/bson/statics.dart

        _Pid = (Math.random() * 0xFFFF).floor().toInt();
     }
     return _Pid;
-  }  
+  }
+  static Map<String,List<int>> _keys;
+  static Map<String,List<int>> get keys(){
+    if (_keys === null){
+       _keys = new Map<String,List<int>>();
+    }
+    return _keys;
+  }
+  static getKeyUtf8(String key){
+    if (!keys.containsKey(key)){      
+      keys[key] = encodeUtf8(key);    
+    }
+    return keys[key];
+  }
 }

lib/helpers/selector_builder.dart

     map[fieldName] = {"\$mod":[value,0]};
     return this;    
   }
-  SelectorBuilder match(String fieldName, String pattern,[bool multiLine, bool caseInsensitive, bool verbose, bool dotAll, bool unicodeMode, bool localeDependent]){    
-    map[fieldName] = {'\$regex': new BsonRegexp('^str_(5|7|8)17\$',multiLine:multiLine, caseInsensitive:caseInsensitive,
-        verbose:verbose, dotAll:dotAll,unicodeMode:unicodeMode,localeDependent:localeDependent)};
+  SelectorBuilder match(String fieldName, String pattern,[bool multiLine, bool caseInsensitive, bool dotAll, bool extended]){    
+    map[fieldName] = {'\$regex': new BsonRegexp(pattern,multiLine:multiLine, caseInsensitive:caseInsensitive,
+        dotAll:dotAll,extended:extended)};
     return this;    
   }
   SelectorBuilder range(String fieldName, min, max, [bool minInclude=true, bool maxInclude=true]){

tests/CursorTest.dart

 }
 
 main(){
-  setVerboseState();
   group("Cursor tests:", (){
     test("testCursorCreation",testCursorCreation);    
     asyncTest("testCursorClosing",1,testCursorClosing);

tests/DbCollectionTest.dart

 }
 
 main(){
-  setVerboseState();
   group("DbCollection tests:", (){
     asyncTest("testLimit",1,testLimit);
     asyncTest("testSkip",1,testSkip);    

tests/DbCommandTest.dart

 }
 
 main(){
-  setVerboseState();
   group("DBCommand tests:", (){
     test("testDbCommandCreation",testDbCommandCreation);
     test("testPingDbCommand",testPingDbCommand);

tests/SelectorBuilderTest.dart

-#library("mcollection");
+#library("HelperTest");
 #import("../lib/mongo.dart");
+#import("../lib/bson/bson.dart");
 #import("dart:io");
 #import("dart:builtin");
 #import('../third_party/testing/unittest/unittest_vm.dart');
 }
 testSelectorBuilderOnObjectId(){
   ObjectId id = new ObjectId();
-  SelectorBuilder selector = query().eq("_id":);
-  expect(selector is Map && selector.isEmpty()).isTrue();
+  SelectorBuilder selector = query().id(id);
+  expect(selector is Map && selector.isEmpty()).isFalse();
 }
-
 main(){  
   group("DbCollection tests:", (){
     test("testSelectorBuilderCreation",testSelectorBuilderCreation);
+    test("testSelectorBuilderOnObjectId",testSelectorBuilderOnObjectId);    
   });
 }

tests/allTests.dart

 #import("DbCollectionTest.dart",prefix:"dbcollection");
 #import("DbTest.dart",prefix:"dbtest");
 #import("bson/allBsonTests.dart",prefix:"bson");
+#import("SelectorBuilderTest.dart",prefix:"helper");
 main(){
   bson.main();
   connection.main();
   dbcommand.main();
   dbcollection.main();
   dbtest.main();
+  helper.main();  
 }

tests/bson/BsonImplTest.dart

 #library("bsonimpltest");
 #import('../../third_party/testing/unittest/unittest_vm.dart');
 #import("../../lib/bson/bson.dart");
-testSerializeDesirialize(){
+testSerializeDeserialize(){
   var bson = new BSON();
   var map = {'_id':5, 'a':4};
   Binary buffer = bson.serialize(map);
   Expect.equals(5,buffer.offset);
   buffer.offset = 0;
   root = bson.deserialize(buffer);  
-  Expect.equals(doc1['a'][2],root['a'][2]);
-  Expect.equals(buffer.byteLength(),buffer.offset);
+  Expect.equals(doc1['a'][2],root['a'][2], "doc1['a']");
 }
 main(){
   group("BsonImpl:", (){
-    test("testSerializeDesirialize",testSerializeDesirialize);    
+    test("testSerializeDeserialize",testSerializeDeserialize);    
   });
 }
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.