Source

setuptools / setuptools / tests / test_sdist.py

Full commit
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
# -*- coding: utf-8 -*-
"""sdist tests"""

import locale
import os
import shutil
import sys
import tempfile
import unittest
import unicodedata
import re
from setuptools.tests import environment, test_svn
from setuptools.tests.py26compat import skipIf

from setuptools.compat import StringIO, unicode
from setuptools.tests.py26compat import skipIf
from setuptools.command.sdist import sdist, walk_revctrl
from setuptools.command.egg_info import manifest_maker
from setuptools.dist import Distribution
from setuptools import svn_utils

SETUP_ATTRS = {
    'name': 'sdist_test',
    'version': '0.0',
    'packages': ['sdist_test'],
    'package_data': {'sdist_test': ['*.txt']}
}


SETUP_PY = """\
from setuptools import setup

setup(**%r)
""" % SETUP_ATTRS


if sys.version_info >= (3,):
    LATIN1_FILENAME = 'smörbröd.py'.encode('latin-1')
else:
    LATIN1_FILENAME = 'sm\xf6rbr\xf6d.py'


# Cannot use context manager because of Python 2.4
def quiet():
    global old_stdout, old_stderr
    old_stdout, old_stderr = sys.stdout, sys.stderr
    sys.stdout, sys.stderr = StringIO(), StringIO()

def unquiet():
    sys.stdout, sys.stderr = old_stdout, old_stderr


# Fake byte literals for Python <= 2.5
def b(s, encoding='utf-8'):
    if sys.version_info >= (3,):
        return s.encode(encoding)
    return s


# Convert to POSIX path
def posix(path):
    if sys.version_info >= (3,) and not isinstance(path, str):
        return path.replace(os.sep.encode('ascii'), b('/'))
    else:
        return path.replace(os.sep, '/')


# HFS Plus uses decomposed UTF-8
def decompose(path):
    if isinstance(path, unicode):
        return unicodedata.normalize('NFD', path)
    try:
        path = path.decode('utf-8')
        path = unicodedata.normalize('NFD', path)
        path = path.encode('utf-8')
    except UnicodeError:
        pass # Not UTF-8
    return path


class TestSdistTest(unittest.TestCase):

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp()
        f = open(os.path.join(self.temp_dir, 'setup.py'), 'w')
        f.write(SETUP_PY)
        f.close()
        # Set up the rest of the test package
        test_pkg = os.path.join(self.temp_dir, 'sdist_test')
        os.mkdir(test_pkg)
        # *.rst was not included in package_data, so c.rst should not be
        # automatically added to the manifest when not under version control
        for fname in ['__init__.py', 'a.txt', 'b.txt', 'c.rst']:
            # Just touch the files; their contents are irrelevant
            open(os.path.join(test_pkg, fname), 'w').close()

        self.old_cwd = os.getcwd()
        os.chdir(self.temp_dir)

    def tearDown(self):
        os.chdir(self.old_cwd)
        shutil.rmtree(self.temp_dir)

    def test_package_data_in_sdist(self):
        """Regression test for pull request #4: ensures that files listed in
        package_data are included in the manifest even if they're not added to
        version control.
        """

        dist = Distribution(SETUP_ATTRS)
        dist.script_name = 'setup.py'
        cmd = sdist(dist)
        cmd.ensure_finalized()

        # squelch output
        quiet()
        try:
            cmd.run()
        finally:
            unquiet()

        manifest = cmd.filelist.files
        self.assertTrue(os.path.join('sdist_test', 'a.txt') in manifest)
        self.assertTrue(os.path.join('sdist_test', 'b.txt') in manifest)
        self.assertTrue(os.path.join('sdist_test', 'c.rst') not in manifest)

    def test_manifest_is_written_with_utf8_encoding(self):
        # Test for #303.
        dist = Distribution(SETUP_ATTRS)
        dist.script_name = 'setup.py'
        mm = manifest_maker(dist)
        mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
        os.mkdir('sdist_test.egg-info')

        # UTF-8 filename
        filename = os.path.join('sdist_test', 'smörbröd.py')

        # Add UTF-8 filename and write manifest
        quiet()
        try:
            mm.run()
            mm.filelist.files.append(filename)
            mm.write_manifest()
        finally:
            unquiet()

        manifest = open(mm.manifest, 'rbU')
        contents = manifest.read()
        manifest.close()

        # The manifest should be UTF-8 encoded
        try:
            u_contents = contents.decode('UTF-8')
        except UnicodeDecodeError:
            e = sys.exc_info()[1]
            self.fail(e)

        # The manifest should contain the UTF-8 filename
        if sys.version_info < (3,):
            fs_enc = sys.getfilesystemencoding()
            filename = filename.decode(fs_enc)

        self.assertTrue(posix(filename) in u_contents)

    # Python 3 only
    if sys.version_info >= (3,):

        def test_write_manifest_allows_utf8_filenames(self):
            # Test for #303.
            dist = Distribution(SETUP_ATTRS)
            dist.script_name = 'setup.py'
            mm = manifest_maker(dist)
            mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
            os.mkdir('sdist_test.egg-info')

            # UTF-8 filename
            filename = os.path.join(b('sdist_test'), b('smörbröd.py'))

            # Add filename and write manifest
            quiet()
            try:
                mm.run()
                u_filename = filename.decode('utf-8')
                mm.filelist.files.append(u_filename)
                # Re-write manifest
                mm.write_manifest()
            finally:
                unquiet()

            manifest = open(mm.manifest, 'rbU')
            contents = manifest.read()
            manifest.close()

            # The manifest should be UTF-8 encoded
            try:
                contents.decode('UTF-8')
            except UnicodeDecodeError:
                e = sys.exc_info()[1]
                self.fail(e)

            # The manifest should contain the UTF-8 filename
            self.assertTrue(posix(filename) in contents)

            # The filelist should have been updated as well
            self.assertTrue(u_filename in mm.filelist.files)

        def test_write_manifest_skips_non_utf8_filenames(self):
            # Test for #303.
            dist = Distribution(SETUP_ATTRS)
            dist.script_name = 'setup.py'
            mm = manifest_maker(dist)
            mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
            os.mkdir('sdist_test.egg-info')

            # Latin-1 filename
            filename = os.path.join(b('sdist_test'), LATIN1_FILENAME)

            # Add filename with surrogates and write manifest
            quiet()
            try:
                mm.run()
                u_filename = filename.decode('utf-8', 'surrogateescape')
                mm.filelist.files.append(u_filename)
                # Re-write manifest
                mm.write_manifest()
            finally:
                unquiet()

            manifest = open(mm.manifest, 'rbU')
            contents = manifest.read()
            manifest.close()

            # The manifest should be UTF-8 encoded
            try:
                contents.decode('UTF-8')
            except UnicodeDecodeError:
                e = sys.exc_info()[1]
                self.fail(e)

            # The Latin-1 filename should have been skipped
            self.assertFalse(posix(filename) in contents)

            # The filelist should have been updated as well
            self.assertFalse(u_filename in mm.filelist.files)

    def test_manifest_is_read_with_utf8_encoding(self):
        # Test for #303.
        dist = Distribution(SETUP_ATTRS)
        dist.script_name = 'setup.py'
        cmd = sdist(dist)
        cmd.ensure_finalized()

        # Create manifest
        quiet()
        try:
            cmd.run()
        finally:
            unquiet()

        # Add UTF-8 filename to manifest
        filename = os.path.join(b('sdist_test'), b('smörbröd.py'))
        cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
        manifest = open(cmd.manifest, 'ab')
        manifest.write(b('\n')+filename)
        manifest.close()

        # The file must exist to be included in the filelist
        open(filename, 'w').close()

        # Re-read manifest
        cmd.filelist.files = []
        quiet()
        try:
            cmd.read_manifest()
        finally:
            unquiet()

        # The filelist should contain the UTF-8 filename
        if sys.version_info >= (3,):
            filename = filename.decode('utf-8')
        self.assertTrue(filename in cmd.filelist.files)

    # Python 3 only
    if sys.version_info >= (3,):

        def test_read_manifest_skips_non_utf8_filenames(self):
            # Test for #303.
            dist = Distribution(SETUP_ATTRS)
            dist.script_name = 'setup.py'
            cmd = sdist(dist)
            cmd.ensure_finalized()

            # Create manifest
            quiet()
            try:
                cmd.run()
            finally:
                unquiet()

            # Add Latin-1 filename to manifest
            filename = os.path.join(b('sdist_test'), LATIN1_FILENAME)
            cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
            manifest = open(cmd.manifest, 'ab')
            manifest.write(b('\n')+filename)
            manifest.close()

            # The file must exist to be included in the filelist
            open(filename, 'w').close()

            # Re-read manifest
            cmd.filelist.files = []
            quiet()
            try:
                try:
                    cmd.read_manifest()
                except UnicodeDecodeError:
                    e = sys.exc_info()[1]
                    self.fail(e)
            finally:
                unquiet()

            # The Latin-1 filename should have been skipped
            filename = filename.decode('latin-1')
            self.assertFalse(filename in cmd.filelist.files)

    @skipIf(sys.version_info >= (3,) and locale.getpreferredencoding() != 'UTF-8',
            'Unittest fails if locale is not utf-8 but the manifests is recorded correctly')
    def test_sdist_with_utf8_encoded_filename(self):
        # Test for #303.
        dist = Distribution(SETUP_ATTRS)
        dist.script_name = 'setup.py'
        cmd = sdist(dist)
        cmd.ensure_finalized()

        # UTF-8 filename
        filename = os.path.join(b('sdist_test'), b('smörbröd.py'))
        open(filename, 'w').close()

        quiet()
        try:
            cmd.run()
        finally:
            unquiet()

        if sys.platform == 'darwin':
            filename = decompose(filename)

        if sys.version_info >= (3,):
            fs_enc = sys.getfilesystemencoding()

            if sys.platform == 'win32':
                if fs_enc == 'cp1252':
                    # Python 3 mangles the UTF-8 filename
                    filename = filename.decode('cp1252')
                    self.assertTrue(filename in cmd.filelist.files)
                else:
                    filename = filename.decode('mbcs')
                    self.assertTrue(filename in cmd.filelist.files)
            else:
                filename = filename.decode('utf-8')
                self.assertTrue(filename in cmd.filelist.files)
        else:
            self.assertTrue(filename in cmd.filelist.files)

    def test_sdist_with_latin1_encoded_filename(self):
        # Test for #303.
        dist = Distribution(SETUP_ATTRS)
        dist.script_name = 'setup.py'
        cmd = sdist(dist)
        cmd.ensure_finalized()

        # Latin-1 filename
        filename = os.path.join(b('sdist_test'), LATIN1_FILENAME)
        open(filename, 'w').close()
        self.assertTrue(os.path.isfile(filename))

        quiet()
        try:
            cmd.run()
        finally:
            unquiet()

        if sys.version_info >= (3,):
            #not all windows systems have a default FS encoding of cp1252
            if sys.platform == 'win32':
                # Latin-1 is similar to Windows-1252 however
                # on mbcs filesys it is not in latin-1 encoding
                fs_enc = sys.getfilesystemencoding()
                if fs_enc == 'mbcs':
                    filename = filename.decode('mbcs')
                else:
                    filename = filename.decode('latin-1')

                self.assertTrue(filename in cmd.filelist.files)
            else:
                # The Latin-1 filename should have been skipped
                filename = filename.decode('latin-1')
                self.assertFalse(filename in cmd.filelist.files)
        else:
            # No conversion takes place under Python 2 and the file
            # is included. We shall keep it that way for BBB.
            self.assertTrue(filename in cmd.filelist.files)


class TestDummyOutput(environment.ZippedEnvironment):

    def setUp(self):
        self.datafile = os.path.join('setuptools', 'tests',
                                     'svn_data', "dummy.zip")
        self.dataname = "dummy"
        super(TestDummyOutput, self).setUp()

    def _run(self):
        code, data = environment.run_setup_py(["sdist"],
                                              pypath=self.old_cwd,
                                              data_stream=0)
        if code:
            info = "DIR: " + os.path.abspath('.')
            info += "\n  SDIST RETURNED: %i\n\n" % code
            info += data
            raise AssertionError(info)

        datalines = data.splitlines()

        possible = (
            "running sdist",
            "running egg_info",
            "creating dummy\.egg-info",
            "writing dummy\.egg-info",
            "writing top-level names to dummy\.egg-info",
            "writing dependency_links to dummy\.egg-info",
            "writing manifest file 'dummy\.egg-info",
            "reading manifest file 'dummy\.egg-info",
            "reading manifest template 'MANIFEST\.in'",
            "writing manifest file 'dummy\.egg-info",
            "creating dummy-0.1.1",
            "making hard links in dummy-0\.1\.1",
            "copying files to dummy-0\.1\.1",
            "copying \S+ -> dummy-0\.1\.1",
            "copying dummy",
            "copying dummy\.egg-info",
            "hard linking \S+ -> dummy-0\.1\.1",
            "hard linking dummy",
            "hard linking dummy\.egg-info",
            "Writing dummy-0\.1\.1",
            "creating dist",
            "creating 'dist",
            "Creating tar archive",
            "running check",
            "adding 'dummy-0\.1\.1",
            "tar .+ dist/dummy-0\.1\.1\.tar dummy-0\.1\.1",
            "gzip .+ dist/dummy-0\.1\.1\.tar",
            "removing 'dummy-0\.1\.1' \\(and everything under it\\)",
        )

        print("    DIR: " + os.path.abspath('.'))
        for line in datalines:
            found = False
            for pattern in possible:
                if re.match(pattern, line):
                    print("   READ: " + line)
                    found = True
                    break
            if not found:
                raise AssertionError("Unexpexected: %s\n-in-\n%s"
                                     % (line, data))

        return data

    def test_sources(self):
        self._run()


class TestSvn(environment.ZippedEnvironment):

    def setUp(self):
        version = svn_utils.SvnInfo.get_svn_version()
        if not version:  # None or Empty
            return

        self.base_version = tuple([int(x) for x in version.split('.')][:2])

        if not self.base_version:
            raise ValueError('No SVN tools installed')
        elif self.base_version < (1, 3):
            raise ValueError('Insufficient SVN Version %s' % version)
        elif self.base_version >= (1, 9):
            #trying the latest version
            self.base_version = (1, 8)

        self.dataname = "svn%i%i_example" % self.base_version
        self.datafile = os.path.join('setuptools', 'tests',
                                     'svn_data', self.dataname + ".zip")
        super(TestSvn, self).setUp()

    @skipIf(not test_svn._svn_check, "No SVN to text, in the first place")
    def test_walksvn(self):
        if self.base_version >= (1, 6):
            folder2 = 'third party2'
            folder3 = 'third party3'
        else:
            folder2 = 'third_party2'
            folder3 = 'third_party3'

        #TODO is this right
        expected = set([
            os.path.join('a file'),
            os.path.join(folder2, 'Changes.txt'),
            os.path.join(folder2, 'MD5SUMS'),
            os.path.join(folder2, 'README.txt'),
            os.path.join(folder3, 'Changes.txt'),
            os.path.join(folder3, 'MD5SUMS'),
            os.path.join(folder3, 'README.txt'),
            os.path.join(folder3, 'TODO.txt'),
            os.path.join(folder3, 'fin'),
            os.path.join('third_party', 'README.txt'),
            os.path.join('folder', folder2, 'Changes.txt'),
            os.path.join('folder', folder2, 'MD5SUMS'),
            os.path.join('folder', folder2, 'WatashiNiYomimasu.txt'),
            os.path.join('folder', folder3, 'Changes.txt'),
            os.path.join('folder', folder3, 'fin'),
            os.path.join('folder', folder3, 'MD5SUMS'),
            os.path.join('folder', folder3, 'oops'),
            os.path.join('folder', folder3, 'WatashiNiYomimasu.txt'),
            os.path.join('folder', folder3, 'ZuMachen.txt'),
            os.path.join('folder', 'third_party', 'WatashiNiYomimasu.txt'),
            os.path.join('folder', 'lalala.txt'),
            os.path.join('folder', 'quest.txt'),
            # The example will have a deleted file
            #  (or should) but shouldn't return it
        ])
        self.assertEqual(set(x for x in walk_revctrl()), expected)


def test_suite():
    return unittest.defaultTestLoader.loadTestsFromName(__name__)