1. Hideya OHASHI
  2. icedfs

Commits

Hideya OHASHI  committed 07548b9

check: auto replication

  • Participants
  • Parent commits 6a2abe6
  • Branches default

Comments (0)

Files changed (2)

File sdfs.rb

View file
  • Ignore whitespace
   CONFIGNAME = 'config'
   DIGEST_METHOD = Digest::SHA256
   MD5 = Digest::MD5
+  REP = 2
 
   def initialize
     @storagetable = []
     return false if destdir != '.' and file?(destdir)
     dest += '/' + File.basename(src) if dir?(dest)
     digest = DIGEST_METHOD.file(src)
-    rep = 2
+    rep = REP
     datanodes(digest.to_s).each do |datanode|
       next unless datanode.put(src, digest.to_s)
       # TODO take a snapshot to namenodes if datanode is removable
       rep -= 1
       break if rep == 0
     end
-    return false if rep == 2
+    return false if rep == REP
     puts 'warning: replication' if rep > 0
     index = ls(:raw => true)
     searchptn = /^#{dest} /
       chunks = l.split
       fname = chunks.shift # TODO filename contains space
       chunks.each do |chunk|
-        count = 2
+        count = REP
         rep = 0
+        repnodes = []
+        hasnode = nil
         datanodes(chunk).each do |datanode|
           if datanode.exist?(chunk)
             rep += 1
+            hasnode = datanode unless hasnode
           else
             if count > 0
               unless quiet
                 puts "warning: #{datanode} should have #{chunk[0,12]}"
               end
+              if datanode.online?
+                repnodes << datanode
+              end
               unbalance += 1
             end
           end
         if rep == 0
           puts "error: missing #{chunk[0,12]}" unless quiet
           missing += 1
-        elsif rep < 2
+        elsif rep < REP
           puts "warning: no replication #{chunk[0,12]}" unless quiet
           replication += 1
-        elsif rep > 2
+          data = hasnode.get(chunk)
+          if data
+            tmp = Tempfile.new('sdfs')
+            tmp.close
+            tmpfname = tmp.path
+            FileUtils.rm_f(tmpfname)
+            open(tmpfname, 'wb') do |f|
+              f.write(data)
+            end
+            repnodes.each do |repnode|
+              repnode.put(tmpfname, chunk)
+            end
+          end
+        elsif rep > REP
           overrep += 1
         end
       end
       next if file == 'placeholder'
       if file != digest
         unless quiet
-          puts "#{@url} has a bad chunk." # TODO fix when?
+          puts "#{@url} has a bad chunk."
           puts file[0, 12]
           puts digest[0, 12]
         end
           digest, file = l.split
           unless file.index(digest)
             unless quiet
-              puts "#{@url} has a bad chunk." # TODO fix when?
-              puts file
-              puts digest
+              puts "#{@url} has a bad chunk."
+              puts file[0, 12]
+              puts digest[0, 12]
             end
+            ssh("mkdir -p #{@path}/badchunk")
+            ssh("mv #{@path}/chunk/#{file} #{@path}/badchunk")
             result = false
           end
         end

File test.rb

View file
  • Ignore whitespace
       f.write('Z')
     end
     assert(!@dfs.check(true))
-    assert_nil(@dfs.get('a', false))
+    text = @dfs.get('a', false)
+    assert_equal(Digest::MD5.hexdigest("z\n"), Digest::MD5.hexdigest(text))
   end
 
   def test_offline