Commits

Liam Staskawicz  committed a970473

* first stab at GridFS

  • Participants
  • Parent commits cd13f59

Comments (0)

Files changed (4)

File fan/gridfs/GridFS.fan

+////////////////////////////////////////////////////////////////////////////////
+//
+//  Copyright 2010 Liam Staskawicz
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+
+**
+**  GridFS
+**
+const class GridFS 
+{
+  static const Int DEFAULT_CHUNK_SIZE := 1024 * 256
+  const DB db
+  const Str root
+  
+  new make(DB db, Str root := "fs")
+  {
+    this.db = db
+    this.root = root
+    db[root].createIndex([["files_id": Index.ASCENDING], ["n": Index.ASCENDING]]);
+  }
+  
+  Cursor fileList(Str:Obj? query := [:])
+  {
+    return filesColl.find(query)  // todo - .sort(["filename": 1]);
+  }
+  
+  internal Collection filesColl()
+  {
+    return db["${this.root}.files"]
+  }
+  
+  internal Collection chunkColl()
+  {
+    return db["${this.root}.chunks"]
+  }
+  
+  **
+  ** Create a new file.
+  ** You'll need to call 'GridFSFile.save' to actually store any data.
+  **
+  GridFSFile createFile(Str name, MimeType mt := MimeType.fromStr("text/plain"), Obj _id := ObjectID())
+  {
+    return GridFSFile(this, name, mt, _id)
+  }
+  
+  GridFSFile? findOne(Str:Obj? query := [:])
+  {
+    o := filesColl.findOne(query)
+    if (o == null) return null
+    return fix(o)
+  }
+  
+  GridFSFile? findOneByName(Str filename)
+  {
+    return findOne(["filename":filename])
+  }
+  
+  private GridFSFile fix(Str:Obj? o)
+  {
+    return GridFSFile(this, o["filename"], MimeType.fromStr(o["contentType"]), o["_id"]) {
+      _size = o["length"]
+      createdOn = o["uploadDate"]
+      md5 = o["md5"]
+      if(o.containsKey("metadata")) metadata = o["metadata"] 
+      savedChunks = true // so additional data can't be uploaded
+    }
+  }
+  
+  Void remove(Str:Obj? query := [:])
+  {
+    filesColl.find(query).each |o| {
+      removeID(o["_id"])
+    }
+  }
+  
+  Void removeID(ObjectID id)
+  {
+    filesColl.remove(["_id": id])
+    chunkColl.remove(["files_id": id])
+  }
+  
+  Void removeStr(Str filename)
+  {
+    remove(["filename": filename])
+  }
+  
+  Void removeAll()
+  {
+    filesColl.remove()
+    chunkColl.remove()
+  }
+}
+
+

File fan/gridfs/GridFSFile.fan

+////////////////////////////////////////////////////////////////////////////////
+//
+//  Copyright 2010 Liam Staskawicz
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+
+**
+**  GridFSFile
+**
+class GridFSFile
+{
+  const Obj _id
+  Str name
+  MimeType mimeType
+  [Str:Obj?] metadata := [:]
+  
+  internal Int _size := 0
+  internal GridFS gfs
+  internal Str md5 := ""
+  internal DateTime createdOn := DateTime.now
+  internal Bool savedChunks := false
+  
+  new make(GridFS gfs, Str name, MimeType mt := MimeType.fromStr("text/plain"), Obj _id := ObjectID())
+  {
+    this._id = _id
+    this.gfs = gfs
+    this.name = name
+    this.mimeType = mt
+  }
+  
+  **
+  ** Return the size in bytes of this file
+  **
+  Int size()
+  {
+    return _size
+  }
+  
+  **
+  ** Confirm, by checking with the DB, that this file is valid.
+  ** note - this is not implemented, so will always return false for the moment, 
+  ** even though the file is most likely valid (assuming no IOErrs, etc)
+  **
+  Bool isValid()
+  {
+    if (md5.isEmpty) return false
+    res := gfs.db.command(["filemd5": _id])
+    if(!DB.cmdOk(res))
+      throw MongoOpErr("GridFSFile.isValid err - $res")
+    return res["md5"] == this.md5
+  }
+  
+  **
+  ** Store data in this file.
+  ** If this file has been saved before, only the metadata will
+  ** be updated - returns false in this case, otherwise
+  ** saves 'ins' to this file and returns true.
+  **
+  Bool save(InStream ins)
+  {
+    didSaveRaw := false
+    if(!savedChunks){
+      saveRawData(ins)
+      didSaveRaw = true
+    }
+    gfs.filesColl.update(["_id":_id], this.toMap, true)
+    return didSaveRaw
+  }
+  
+  Int saveRawData(InStream ins)
+  {
+    b := Buf(GridFS.DEFAULT_CHUNK_SIZE)
+    total := 0
+    chunkNum := 0
+    
+    // todo - calculate md5 as we go...something like the following in java
+    // MessageDigest md = _md5Pool.get();
+    // md.reset();
+    // DigestInputStream in = new DigestInputStream( _in , md );
+    
+    more := true
+    Int? v
+    while (more) {
+      while (b.size < b.capacity) {
+        v = ins.readBuf(b, b.capacity - b.size)
+        if(v == null || v == 0) { // should only have to check for null. fix after 1.0.49 is released...was a problem with Str.in
+          more = false
+          ins.close
+          break
+        }
+        else
+          total += v
+      }
+      
+      gfs.chunkColl.save(["files_id": _id, "n": chunkNum++, "data": b.flip])
+      b.clear
+    }
+    _size = total
+    savedChunks = true
+    return chunkNum
+  }
+  
+  Int numChunks()
+  {
+    f := _size.toFloat / GridFS.DEFAULT_CHUNK_SIZE.toFloat
+    return f.ceil.toInt
+  }
+  
+  Int write(OutStream out)
+  {
+    (0..<numChunks).each |i| { out.writeBuf(getChunk(i)) }
+    return size
+  }
+  
+  Void remove()
+  {
+    gfs.filesColl.remove(["_id": _id])
+    gfs.chunkColl.remove(["files_id": _id])
+  }
+  
+  internal Buf getChunk(Int i)
+  {
+    chunk := gfs.chunkColl.findOne(["files_id": _id, "n": i])
+    if (chunk == null)
+      throw MongoOpErr("can't find a chunk!  file id: $_id chunk: $i")
+    return chunk["data"]
+  }
+  
+  Str:Obj toMap()
+  {
+    Str:Obj m := [:] { ordered = true }
+    m["_id"] = _id
+    m["filename"] = name
+    m["contentType"] = mimeType.toStr
+    m["length"] = _size
+    m["chunkSize"] = GridFS.DEFAULT_CHUNK_SIZE
+    m["uploadDate"] = createdOn
+    // if(false) m["aliases"] = ["test", "test2"] // this isn't support in other drivers yet...
+    if(metadata.size > 0) m["metadata"] = metadata
+    m["md5"] = md5
+    return m
+  }
+  
+  override Str toStr()
+  {
+    return this.toMap.toStr
+  }
+}
+
+**
+** TODO - add ChunkInStream and ChunkOutStream to read/write directly into GridFS
+**
+
 **
 
 @podDepends = [Depend("sys 1.0"), Depend("inet 1.0")]
-@podSrcDirs = [`fan/`, `fan/bson/`, `test/`]
+@podSrcDirs = [`fan/`, `fan/bson/`, `fan/gridfs/`, `test/`]
 
 pod mongo {}

File test/GridFSTest.fan

+////////////////////////////////////////////////////////////////////////////////
+//
+//  Copyright 2010 Liam Staskawicz
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+
+**
+**  GridFSTest
+**
+class GridFSTest : MongoTest
+{
+  override Void setup()
+  {
+  
+  }
+
+  override Void teardown()
+  {
+  
+  }
+  
+  Void testBasic()
+  {
+    gfs := GridFS(db)
+    gfs.removeAll
+    verifyEq(0, gfs.fileList.count, "gridfs isn't empty starting test")
+    f := gfs.createFile("tester1")
+    data := "allo there"
+    f.save(data.in)
+    verifyEq(f.size, data.size)
+    
+    bigfilepath := "/Users/liam/Documents/mtcode/fan/fantomongo/test/Ruckus.mp3"
+    bigfile := File(Uri.fromStr(bigfilepath))
+    if(!bigfile.exists)
+      throw Err("please update fantomongo::test::GridFSTest with a large file that exists on your machine")
+    verify(bigfile.size > GridFS.DEFAULT_CHUNK_SIZE, "must be large enough to test multiple chunks")
+    f2 := gfs.createFile("bigfiletest", bigfile.mimeType)
+    f2.save(bigfile.in)
+    verifyEq(bigfile.size, f2.size)
+    
+    c := gfs.fileList
+    verifyEq(2, c.count)
+    c.each |o| {
+      verify(o.keys.containsAll(["_id", "filename", "contentType", "length", "chunkSize", "uploadDate"]))
+      // TODO - also check for md5 when implemented
+    }
+    
+    f3 := gfs.findOneByName("tester1")
+    verifyNotNull(f3)
+    sb := Buf() // have to read to a Buf first since StrBuf won't allow writeBuf() on it
+    f3.write(sb.out)
+    sv := sb.readChars(sb.flip.size)
+    
+    path := bigfile.path.dup
+    path[path.size - 1] = "ReadBack${bigfile.name}"
+    leading := bigfile.uri.isPathAbs ? File.sep : "" // need to re-add the leading slash?
+    readbackfile := File(Uri.fromStr("${leading}${path.join(File.sep)}"))
+    verify(readbackfile.parent == bigfile.parent)
+    fmp3 := gfs.findOneByName("bigfiletest")
+    verifyNotNull(fmp3)
+    fmp3.write(readbackfile.out)
+    verifyEq(readbackfile.mimeType, bigfile.mimeType)
+    verifyEq(readbackfile.size, bigfile.size)
+    
+    gfs.removeAll
+  }
+  
+}
+
+
+