Source

pyLoad / module / plugins / hooks / ExtractArchive.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import os
from os import remove, chmod
from os.path import exists, basename, isfile, isdir
from traceback import print_exc
from copy import copy


# monkey patch bug in python 2.6 and lower
# see http://bugs.python.org/issue6122
# http://bugs.python.org/issue1236
# http://bugs.python.org/issue1731717
if sys.version_info < (2, 7) and os.name != "nt":
    from subprocess import Popen
    import errno

    def _eintr_retry_call(func, *args):
        while True:
            try:
                return func(*args)
            except OSError, e:
                if e.errno == errno.EINTR:
                    continue
                raise

    def wait(self):
        """Wait for child process to terminate.  Returns returncode
        attribute."""
        if self.returncode is None:
            try:
                pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
            except OSError, e:
                if e.errno != errno.ECHILD:
                    raise
                    # This happens if SIGCLD is set to be ignored or waiting
                # for child processes has otherwise been disabled for our
                # process.  This child is dead, we can't get the status.
                sts = 0
            self._handle_exitstatus(sts)
        return self.returncode

    Popen.wait = wait

if os.name != "nt":
    from os import chown
    from pwd import getpwnam
    from grp import getgrnam

from module.plugins.Hook import Hook, threaded, Expose
from module.utils import save_join


class ArchiveError(Exception):
    pass


class CRCError(Exception):
    pass


class WrongPassword(Exception):
    pass


class ExtractArchive(Hook):
    """
    Provides: unrarFinished (folder, filename)
    """
    __name__ = "ExtractArchive"
    __version__ = "0.1"
    __description__ = "Extract different kind of archives"
    __config__ = [("activated", "bool", "Activated", True),
        ("fullpath", "bool", "Extract full path", True),
        ("overwrite", "bool", "Overwrite files", True),
        ("passwordfile", "file", "password file", "unrar_passwords.txt"),
        ("deletearchive", "bool", "Delete archives when done", False),
        ("destination", "folder", "Extract files to", ""),
        ("queue", "bool", "Wait for all downloads to be fninished", True),
        ("renice", "int", "CPU Priority", 0), ]
    __author_name__ = ("pyload Team")
    __author_mail__ = ("admin<at>pyload.org")

    event_list = ["allDownloadsProcessed"]

    def setup(self):
        self.plugins = []
        self.passwords = []
        names = []

        for p in ("UnRar", "UnZip"):
            try:
                module = self.core.pluginManager.getInternalModule(p)
                klass = getattr(module, p)
                if klass.checkDeps():
                    names.append(p)
                    self.plugins.append(klass)

            except OSError, e:
                if e.errno == 2:
                    self.logInfo(_("No %s installed") % p)
                else:
                    self.logWarning(_("Could not activate %s") % p, str(e))
                    if self.core.debug:
                        print_exc()

            except Exception, e:
                self.logWarning(_("Could not activate %s") % p, str(e))
                if self.core.debug:
                    print_exc()

        if names:
            self.logInfo(_("Activated") + " " + " ".join(names))
        else:
            self.logInfo(_("No Extract plugins activated"))

        # queue with package ids
        self.queue = []

    @Expose
    def extractPackage(self, id):
        """ Extract package with given id"""
        self.manager.startThread(self.extract, [id])

    def packageFinished(self, pypack):
        if self.getConfig("queue"):
            self.logInfo(_("Package %s queued for later extracting") % pypack.name)
            self.queue.append(pypack.id)
        else:
            self.manager.startThread(self.extract, [pypack.id])


    @threaded
    def allDownloadsProcessed(self, thread):
        local = copy(self.queue)
        del self.queue[:]
        self.extract(local, thread)


    def extract(self, ids, thread=None):
        # reload from txt file
        self.reloadPasswords()

        # dl folder
        dl = self.config['general']['download_folder']

        extracted = []

        #iterate packages -> plugins -> targets
        for pid in ids:
            p = self.core.files.getPackage(pid)
            self.logInfo(_("Check package %s") % p.name)
            if not p: continue

            # determine output folder
            out = save_join(dl, p.folder, "")
            # force trailing slash

            if self.getConfig("destination") and self.getConfig("destination").lower() != "none":
                if exists(self.getConfig("destination")):
                    out = save_join(dl, p.folder, self.getConfig("destination"), "")
                    #relative to package folder if destination is relative, otherwise absolute path overwrites them

            files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()]

            # check as long there are unseen files
            while files_ids:
                new_files_ids = []

                for plugin in self.plugins:
                    targets = plugin.getTargets(files_ids)
                    if targets: self.logDebug("Targets: %s" % targets)
                    for target, fid in targets:
                        if target in extracted:
                            self.logDebug(basename(target), "skipped")
                            continue
                        extracted.append(target) #prevent extracting same file twice

                        klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"),
                            self.getConfig("renice"))
                        klass.init()

                        self.logInfo(basename(target), _("Extract to %s") % out)
                        new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread)
                        self.logDebug("Extracted: %s" % new_files)
                        self.setPermissions(new_files)

                        for file in new_files:
                            if not exists(file):
                                self.logDebug("new file %s does not exists" % file)
                                continue
                            if isfile(file):
                                new_files_ids.append((file, fid)) #append as new target

                files_ids = new_files_ids # also check extracted files


    def startExtracting(self, plugin, fid, passwords, thread):
        pyfile = self.core.files.getFile(fid)
        if not pyfile: return []

        pyfile.setCustomStatus(_("extracting"))
        thread.addActive(pyfile) #keep this file until everything is done

        try:
            progress = lambda x: pyfile.setProgress(x)
            success = False

            if not plugin.checkArchive():
                plugin.extract(progress)
                success = True
            else:
                self.logInfo(basename(plugin.file), _("Password protected"))
                self.logDebug("Passwords: %s" % str(passwords))

                pwlist = copy(self.getPasswords())
                #remove already supplied pws from list (only local)
                for pw in passwords:
                    if pw in pwlist: pwlist.remove(pw)

                for pw in passwords + pwlist:
                    try:
                        self.logDebug("Try password: %s" % pw)
                        if plugin.checkPassword(pw):
                            plugin.extract(progress, pw)
                            self.addPassword(pw)
                            success = True
                            break
                    except WrongPassword:
                        self.logDebug("Password was wrong")

            if not success:
                self.logError(basename(plugin.file), _("Wrong password"))
                return []

            if self.core.debug:
                self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles()))

            if self.getConfig("deletearchive"):
                files = plugin.getDeleteFiles()
                self.logInfo(_("Deleting %s files") % len(files))
                for f in files:
                    if exists(f): remove(f)
                    else: self.logDebug("%s does not exists" % f)

            self.logInfo(basename(plugin.file), _("Extracting finished"))
            self.manager.dispatchEvent("unrarFinished", plugin.out, plugin.file)

            return plugin.getExtractedFiles()


        except ArchiveError, e:
            self.logError(basename(plugin.file), _("Archive Error"), str(e))
        except CRCError:
            self.logError(basename(plugin.file), _("CRC Mismatch"))
        except Exception, e:
            if self.core.debug:
                print_exc()
            self.logError(basename(plugin.file), _("Unknown Error"), str(e))

        return []

    @Expose
    def getPasswords(self):
        """ List of saved passwords """
        return self.passwords


    def reloadPasswords(self):
        pwfile = self.getConfig("passwordfile")
        if not exists(pwfile):
            open(pwfile, "wb").close()

        passwords = []
        f = open(pwfile, "rb")
        for pw in f.read().splitlines():
            passwords.append(pw)
        f.close()

        self.passwords = passwords


    @Expose
    def addPassword(self, pw):
        """  Adds a password to saved list"""
        pwfile = self.getConfig("passwordfile")

        if pw in self.passwords: self.passwords.remove(pw)
        self.passwords.insert(0, pw)

        f = open(pwfile, "wb")
        for pw in self.passwords:
            f.write(pw + "\n")
        f.close()

    def setPermissions(self, files):
        for f in files:
            if not exists(f): continue
            try:
                if self.core.config["permission"]["change_file"]:
                    if isfile(f):
                        chmod(f, int(self.core.config["permission"]["file"], 8))
                    elif isdir(f):
                        chmod(f, int(self.core.config["permission"]["folder"], 8))

                if self.core.config["permission"]["change_dl"] and os.name != "nt":
                    uid = getpwnam(self.config["permission"]["user"])[2]
                    gid = getgrnam(self.config["permission"]["group"])[2]
                    chown(f, uid, gid)
            except Exception, e:
                self.log.warning(_("Setting User and Group failed"), e)

    def archiveError(self, msg):
        raise ArchiveError(msg)

    def wrongPassword(self):
        raise WrongPassword()

    def crcError(self):
        raise CRCError()


class AbtractExtractor:
    @staticmethod
    def checkDeps():
        """ Check if system statisfy dependencies
        :return: boolean
        """
        return True

    @staticmethod
    def getTargets(files_ids):
        """ Filter suited targets from list of filename id tuple list
        :param files_ids: List of filepathes
        :return: List of targets, id tuple list
        """
        raise NotImplementedError


    def __init__(self, m, file, out, fullpath, overwrite, renice):
        """Initialize extractor for specific file

        :param m: ExtractArchive Hook plugin
        :param file: Absolute filepath
        :param out: Absolute path to destination directory
        :param fullpath: extract to fullpath
        :param overwrite: Overwrite existing archives
        :param renice: Renice value
        """
        self.m = m
        self.file = file
        self.out = out
        self.fullpath = fullpath
        self.overwrite = overwrite
        self.renice = renice
        self.files = [] # Store extracted files here


    def init(self):
        """ Initialize additional data structures """
        pass


    def checkArchive(self):
        """Check if password if needed. Raise ArchiveError if integrity is
        questionable.

        :return: boolean
        :raises ArchiveError
        """
        return False

    def checkPassword(self, password):
        """ Check if the given password is/might be correct.
        If it can not be decided at this point return true.

        :param password:
        :return: boolean
        """
        return True

    def extract(self, progress, password=None):
        """Extract the archive. Raise specific errors in case of failure.

        :param progress: Progress function, call this to update status
        :param password password to use
        :raises WrongPassword
        :raises CRCError
        :raises ArchiveError
        :return:
        """
        raise NotImplementedError

    def getDeleteFiles(self):
        """Return list of files to delete, do *not* delete them here.

        :return: List with paths of files to delete
        """
        raise NotImplementedError

    def getExtractedFiles(self):
        """Populate self.files at some point while extracting"""
        return self.files