Commits

Ivo Hinkelmann  committed a59d268

contl10n01: Use Pavels URL list to fetch non pootle langs

  • Participants
  • Parent commits 3d13a6b

Comments (0)

Files changed (1)

File l10ntools/scripts/tool/commands.py

-import os, shutil, re, glob, tempfile, urllib, logging, time, fcntl, threading, signal, sys, string, zipfile, time, datetime
+import os, shutil, re, glob, tempfile, urllib, logging, time, fcntl, threading, signal, sys, string, zipfile, time, datetime, gzip
 from time import gmtime, strftime
 import sdf, pseudo, utils, eiswrapper, eis
 
         self._ftp_server_keyfile = utils.Util.replace_variables(tool_cfg.get("Download Server", "ftp_server_keyfile"), env, tool_cfg, self._log) 
         #self._filesystem       = utils.Util.replace_variables(tool_cfg.get("Download Server", "filesystem"), env, tool_cfg, self._log) 
         self._unzip             =  utils.Util.replace_variables(tool_cfg.get("Download Server", "unzip"), env, tool_cfg, self._log)  
+        self._gunzip            =  utils.Util.replace_variables(tool_cfg.get("Download Server", "gunzip"), env, tool_cfg, self._log)  
+        self._bunzip2           =  utils.Util.replace_variables(tool_cfg.get("Download Server", "bunzip2"), env, tool_cfg, self._log)  
         self._log = log 
         
     def do(self, env):
             cmd = "scp -i %s %s@%s %s" % (self._server_keyfile, self._server_user, self._server_url+":"+self._server_path, self._dest_path)
             self._log.info("cmd="+cmd)
             scp_rc, stdout_v, stderr_v = utils.Util.exe(cmd, self._log)
+        self.wget_sdf()
         # FTP server
-        if not self._ftp_server_url == "" and not self._ftp_server_path == "":
-            cmd = "scp -i %s %s@%s %s" % (self._ftp_server_keyfile, self._ftp_server_user, self._ftp_server_url+":"+self._ftp_server_path, self._dest_path)
-            self._log.info("cmd="+cmd)
-            scp_rc, stdout_v, stderr_v = utils.Util.exe(cmd, self._log)
+        #if not self._ftp_server_url == "" and not self._ftp_server_path == "":
+        #    cmd = "scp -i %s %s@%s %s" % (self._ftp_server_keyfile, self._ftp_server_user, self._ftp_server_url+":"+self._ftp_server_path, self._dest_path)
+        #    self._log.info("cmd="+cmd)
+        #    scp_rc, stdout_v, stderr_v = utils.Util.exe(cmd, self._log)
         #if not self._filesystem == "":
         #    filelist = glob.glob(self._filesystem)
         #    for file in filelist:
         if not self._unzip == "":
             zipfilelist = glob.glob(self._unzip)
             for file in zipfilelist:
-                zfile = zipfile.ZipFile(file)
-                zfile.extractall(self._dest_path)
+                try:
+                    zfile = zipfile.ZipFile(file)
+                    zfile.extractall(self._dest_path)
+                except:
+                    pass
+
+        if not self._bunzip2 == "":
+            zipfilelist = glob.glob(self._bunzip2)
+            for file in zipfilelist:
+                self._log.info("Unzipping "+file)
+                utils.Util.exe_ignore_errors("bunzip2 "+file, self._log, env, True)
+
+        if not self._gunzip == "":
+            zipfilelist = glob.glob(self._gunzip)
+            for file in zipfilelist:
+                zfile = gzip.open(file,"r")
+                try:
+                    lines = [line.strip('\n') for line in zfile.readlines()]           
+                    zfile.close()
+                    os.remove(file[:file.rfind(".gz")])
+                    try:
+                        self._log.info("Unzipping "+file+" to "+file[:file.rfind(".gz")])
+                        f = open(file[:file.rfind(".gz")], "w")
+                        for line in lines:
+                            f.write(line+"\n")
+                        f.close()
+                    except:
+                        self._log.die("Can not write to "+file[:file.rfind(".gz")])
+                    os.remove(file)
+                except IOError:
+                    #self._log.warning("File "+file+" seems to be no gzip file ... skipping")
+                    pass
                 
+        #sys.exit(-1)
+
     def post(self, tool_cfg, state_cfg, env):
         # write status into state_cfg
         if not state_cfg.has_section(self.state_name):
             state_cfg.add_section(self.state_name)
 
+    def wget_sdf(self):
+        try:
+            f = open("sdf.lst","r")
+            lines = [line.strip('\n') for line in f.readlines()]           
+            f.close()
+        except:
+            self._log.die("Can't read sdf file list")
+        
+        for line in lines:
+            if not line.startswith("#"):
+                triple = line.split(" ")
+                if len(triple) >2:
+                    url = triple[2]
+                    try:
+                        self._log.info("Retrieve "+url+" -> "+ self._dest_path + url[url.rfind("/"):])
+                        urllib.urlretrieve(url, self._dest_path + url[url.rfind("/"):])
+                    except urllib.ContentTooShortError,e:
+                        self._log.warning("Downloading "+url+" failed!" + str(e))
+                    except IOError,e:
+                        self._log.warning("Can not copy "+url+" to " + self._dest_path + "\n" + str(e))
+
 class Merge(Command):
     """This step is responsible for checking and merging the sdf files. """
     state_name      = "MERGE"
             if my_log_file.startswith("./"):
                 my_log_file = my_log_file[2:]
             
-            if utils.Util.get_platform == "windows":
+            if utils.Util.get_platform() == "windows":
                 utils.Util.copyfile(self._log_files[0], self._log_windows_path+"/"+env["CWS_WORK_STAMP"]+"_"+my_log_file, self._log)
             else:
                 utils.Util.copyfile(self._log_files[0], self._log_unix_path+"/"+env["CWS_WORK_STAMP"]+"_"+my_log_file, self._log)