Commits

Anonymous committed 09c83d8

first commit

  • Participants

Comments (0)

Files changed (5)

+syntax: glob
+.idea
+windows do not support rsync !!
+rsync like purely in Node.
+
+That's a large task, so the first step is to implement the part I actually need.  Namely:
+
+```
+rsync --archive --delete localdir user@server:remotedir
+```
+
+Usage:
+
+```
+var sftp2sync = require('sftp2sync');
+
+//var root_local = process.env["HOME"] + "/test/out";
+var root_local = "./";
+var root_remote = "/home/xxxxx/public_html";
+var force = false;
+var exclude = [''];
+
+sftp2sync.upload(root_local, root_remote, force, excludes, {
+     
+    //    debug: console.log,
+      host: 'xxx.xxx.xxx.xxx',
+      //host: 'localhost',
+      port: 22,
+      username: 'root',
+      privateKey: require('fs').readFileSync('keys'),
+      passphrase: ''
+      /*
+      password: 'password'
+      */
+    });
+```
+
+ToDo: recursive delete imperfectly
+var Connection = require('ssh2');
+var _     = require('lodash');
+var util = require('util');
+var async = require('async');
+var fs    = require('fs');
+var path  = require('path');
+
+module.exports.upload = function(root_local, root_remote, force, excludes, ssh2opts) {
+    
+    var c = new Connection();
+    c.on('connect', function() {
+        util.log('Connection :: connect');
+    });
+    c.on('ready', function() {
+        util.log('Connection :: ready');
+        c.sftp(function(err, sftp) {
+            if (err) throw err;
+            sftp.on('end', function() {
+                util.log('SFTP :: SFTP session closed');
+            });
+
+            doit(root_local, root_remote, "", "", force, excludes, sftp, function(err) {
+                if (err) throw err;
+                util.log('SFTP :: Handle closed');
+                sftp.end();
+                c.end();
+            });
+        });
+    });
+    c.on('error', function(err) {
+        util.log('Connection :: error :: ' + err);
+    });
+    c.on('end', function() {
+        util.log('Connection :: end');
+    });
+    c.on('close', function(had_error) {
+        util.log('Connection :: close');
+    });
+    c.connect(ssh2opts);
+}
+
+
+var mkdir = function(sftp, dir, attrs, cb) {
+    sftp.mkdir(dir, attrs, function(err) {
+        if (err) {
+            util.log('ERROR MAKING REMOTE DIR ' + dir + ' '+ err);
+            cb(err);
+        } else {
+            util.log('mkdir ' + dir);
+            sftp.setstat(dir, attrs, function(err) {
+                if (err) cb(err); else cb();
+            });
+        }
+    });
+}
+
+var closehandle = function(sftp, handle, remotefile, statz, cb) {
+    sftp.close(handle, function(err) {
+        if (err) {
+            cb(err);
+        } else {
+            sftp.setstat(remotefile, {
+                ctime: statz.ctime,
+                atime: statz.atime,
+                mtime: statz.mtime
+            }, function(err) {
+                if (err) cb(err); else cb();
+            });
+        }
+    });
+}
+
+var doupload = function(sftp, remotefile, localfile, statz, cb) {
+    sftp.open(remotefile, "w", {
+        ctime: statz.ctime,
+        atime: statz.atime,
+        mtime: statz.mtime
+    }, function(err, handle) {
+        if (err) {
+            cb(err);
+        } else {
+            fs.readFile(localfile, function(err, data) {
+                if (err) {
+                    cb(err);
+                } else {
+                    if (data.length === 0) {
+                        closehandle(sftp, handle, remotefile, statz, cb);
+                    } else {
+                        sftp.write(handle, data, 0, data.length, 0, function(err) {
+                            if (err) {
+                                cb(err);
+                            } else {
+                                closehandle(sftp, handle, remotefile, statz, cb);
+                            }
+                        });
+                    }
+                }
+            });
+        }
+    });
+}
+
+var exclude = function (paths, filters) {
+
+    var result = [];
+    filters.forEach(function(filter) {
+        var filterRegex = new RegExp('^' +
+            filter.replace(/\./g, '\\.')
+                .replace(/(\*?)(\*)(?!\*)/g, function(match, prefix) {
+                    if(prefix == '*') {
+                        return match;
+                    }
+                    return '[^\\/]*';
+                })
+                .replace(/\*\*/g, '\.*') + '$'
+            , 'i');
+
+        paths.forEach(function(path) {
+            if(result.indexOf(path) < 0 && !path.match(filterRegex)) {
+                result.push(path);
+            }
+        });
+
+    });
+    return result;
+}
+
+var deleteRecursive = function(sftp, path) {
+    sftp.opendir(path, function(err, fd){
+        sftp.readdir(fd, function(err, files){
+
+            var fl = new Array();
+
+            files.forEach(function(file){
+                if(file.filename !== '.' && file.filename !== '..'){
+                    console.log('1st loop');
+                    console.log(file.filename);
+                    fl.push(file.filename, fl);
+                }
+            });
+
+            console.log('1st end');
+            console.log(fl);
+
+            fl.forEach(function(filename){
+                var curPath = path + "/" + filename;
+                console.log(curPath);
+                sftp.stat(curPath, function(err, stats){
+                    console.log('d err ');
+                    console.log(err);
+                    console.log('d stats');
+                    console.log(stats);
+                    if(!_.isUndefined(stats)){
+                        console.log('in stats')
+                        console.log(stats.isDirectory());
+                        if(stats.isDirectory()) { // recursive
+                            deleteRecursive(sftp, curPath);
+                        }else { // delete file
+                            sftp.unlink(curPath, function(err) {
+                                if (err) util.log('FAILED to unlink '+ curPath
+                                    +' BECAUSE '+ err);
+                            });
+                        }
+                    }
+                });
+            });
+            sftp.rmdir(path, function(err) {
+                if (err) util.log('FAILED to rmdir '+ path
+                    +' BECAUSE '+ err);
+            });
+        });
+    });
+};
+
+var doit = function(root_local, root_remote, path_local, path_remote, force, excludes, sftp, done) {
+
+    var localdir = path.join(root_local, path_local);
+    // util.log(localdir);
+    var statzdir = fs.statSync(localdir);
+    if (! statzdir.isDirectory()) {
+        throw "NOT A DIRECTORY " + localdir;
+    } else {
+
+        //var filez = exclude(fs.readdirSync(localdir), ['\.*']);
+        var filez = exclude(fs.readdirSync(localdir), excludes);
+
+        async.forEachSeries(filez,
+            function(file, cb) {
+                var localfile = path.normalize(path.join(localdir, file));
+                // util.log('TEST ' + localfile +' PATH '+ path_local +' REMOTE '+ path_remote +' FILE '+ file);
+                var statz = fs.statSync(localfile);
+                // util.log(util.inspect(statz));
+                if (statz.isDirectory()) {
+                    /*use sftp to verify the remote directory exists
+                    if not, make the remote directory
+                    once satisfied either way, */
+                    
+                    var remotedir  = root_remote +'/'+ ((path_remote !== "") ? (path_remote+'/'+file) : file);
+                    // util.log('DIR PATH ' + path_local +' REMOTE DIR '+ remotedir);
+                    sftp.stat(remotedir, function(err, attrs) {
+                        if (err) {
+                            // Most likely the error is remote directory not existing
+                            util.log('CREATING REMOTE DIR ' + remotedir);
+                            mkdir(sftp, remotedir, {
+                                ctime: statz.ctime,
+                                atime: statz.atime,
+                                mtime: statz.mtime
+                            },
+                            function(err) {
+                                doit(root_local, root_remote,
+                                     path.join(path_local, file),
+                                     path_remote +'/'+ file,
+                                     force, excludes, sftp, function(err) {
+                                    if (err) cb(err); else cb();
+                                });
+                            });
+                        } else {
+                            // util.log('REMOTE DIR ' + remotedir +' '+ util.inspect(attrs));
+                            doit(root_local, root_remote,
+                                path.join(path_local, file),
+                                path_remote +'/'+ file,
+                                force, excludes, sftp, function(err) {
+                                if (err) cb(err); else cb();
+                            });
+                        }
+                    });
+                } else {
+                    /*use sftp to verify the remote file exists
+                    if not, upload
+                    if it does, and if local differs from remote, upload*/
+                    
+                    // util.log('FILE PATH ' + localfile);
+                    var remotefile  = root_remote
+                            + (path_remote !== "" ? (path_remote +'/') : "/")
+                            + file;
+                    // util.log('root_remote '+ root_remote +' path_remote '+ path_remote +' file '+ file);
+                    sftp.stat(remotefile, function(err, attrs) {
+                        if (err) {
+                            // Most likely the error is that the remote file does not exist
+                            util.log('upload to ' + remotefile);
+                            doupload(sftp, remotefile, localfile, statz, cb);
+                        } else {
+                            // util.log('REMOTE FILE ' + remotefile); // +' '+ util.inspect(attrs));
+                            if (force) {
+                                util.log('upload to ' + remotefile);
+                                doupload(sftp, remotefile, localfile, statz, cb);
+                            } else {
+                                //util.log('not upload to ' + remotefile);
+                                cb();
+                            }
+                        }
+                    });
+                }
+            },
+            function(err) {
+                if (err) {
+                    util.log('ERR ' + err);
+                    done(err);
+                } else {
+                    var localfiles = fs.readdirSync(localdir);
+                    var checkdir = function(sftp, handle, done) {
+                        sftp.readdir(handle, function(err, list) {
+                            if (err) done(err);
+                            else if (!list) done();
+                            else {
+                                list.forEach(function(entry) {
+                                    var existlocal = false;
+                                    localfiles.forEach(function(fn) {
+                                        if (fn === entry.filename) existlocal = true;
+                                    });
+                                    if (!existlocal && entry.filename !== '.' && entry.filename !== '..') {
+                                        sftp.stat(remotedir+'/'+ entry.filename, function(err, stats) {
+                                            if(!err && stats.isFile()){
+                                                util.log('rm '+ remotedir+'/'+ entry.filename);
+                                                sftp.unlink(remotedir+'/'+ entry.filename, function(err) {
+                                                    if (err) util.log('FAILED to unlink '+ remotedir+'/'+ entry.filename
+                                                        +' BECAUSE '+ err);
+                                                });
+                                            }
+
+                                            if(!err && stats.isDirectory()){
+                                                console.log('dir');
+                                                deleteRecursive(sftp, remotedir+'/'+ entry.filename);
+                                            }
+
+
+                                        });
+                                    }
+                                });
+                                checkdir(sftp, handle, done);
+                            }
+                        });
+                    }
+
+                    var remotedir  = root_remote +'/'+ path_remote;
+                    sftp.opendir(remotedir, function(err, handle) {
+                        if (err) done(err);
+                        else checkdir(sftp, handle, done);
+                    });
+                }
+            });
+    }
+}
+{
+  "author": {
+    "name": "Daisuke Yamashita",
+    "email": "yamashita@sena-networks.co.jp",
+    "url": "http://sena-networks.co.jp"
+  },
+  "name": "sftp2sync",
+  "description": "rsync like sftp2",
+  "homepage": "https://bitbucket.org/dai_yamashita/node-sftp2sync",
+  "repository": {
+    "type": "hg",
+    "url": "https://bitbucket.org/dai_yamashita/node-sftp2sync"
+  },
+  "licenses": [
+    {
+      "type": "MIT"
+    }
+  ],
+  "version": "0.0.1",
+  "engines": {
+    "node": ">=0.8.1"
+  },
+  "keywords": [
+    "ssh", "sftp", "deployment", "rsync", "sync"
+  ],
+  "dependencies": {
+    "ssh2": "*",
+    "async": ">=0.1.22",
+    "lodash": "*"
+  },
+  "readmeFilename": "README.md"
+}
+
+var sftp2sync = require('./index.js');
+
+//var root_local = process.env["HOME"] + "/test/out";
+var root_local = "./";
+var root_remote = "/home/xxxxx/public_html";
+var force = false;
+//var exclude = [''];
+var excludes = ['\.*'];
+
+sftp2sync.upload(root_local, root_remote, force, excludes, {
+      
+    //    debug: console.log,
+      host: 'xxx.xxx.xxx.xxx',
+      //host: 'localhost',
+      port: 22,
+      username: 'root',
+      privateKey: require('fs').readFileSync('keys'),
+      passphrase: ''
+      /*
+      password: 'nodejsrules'
+      */
+    });