Snippets
Created by
Doug Freed
last modified
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 | diff --git a/git-restore-mtime b/git-restore-mtime
new file mode 100755
index 0000000..9731d9e
--- /dev/null
+++ b/git-restore-mtime
@@ -0,0 +1,405 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# git-restore-mtime - Change mtime of files based on commit date of last change
+#
+# Copyright (C) 2012 Rodrigo Silva (MestreLion) <linux@rodrigosilva.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. See <http://www.gnu.org/licenses/gpl.html>
+#
+# Change the modification time (mtime) of all files in work tree, based on the
+# date of the most recent commit that modified the file.
+#
+# Useful prior to generating release tarballs, so each file is archived with a
+# date that resembles the date when the file was actually last modified.
+# (assuming the actual modification date and its commit date are close)
+
+# By default ignores all ignored and untracked files, and also refuses to work
+# on trees with uncommited changes.
+
+if __name__ != "__main__":
+ raise ImportError("%s should not be used as a module." % __name__)
+
+import subprocess, shlex
+import sys, os.path
+import logging as logger
+import argparse
+import time
+
+if sys.version_info[:2] >= (3, 0):
+ def text(s):
+ return s.decode('utf-8')
+else:
+ def text(s):
+ return s
+
+parser = argparse.ArgumentParser(
+ description='Restore original modification time of files based on '
+ 'the date of the most recent commit that modified them. '
+ 'Useful when generating release tarballs.')
+
+parser.add_argument('--quiet', '-q',
+ action="store_true",
+ help='suppress informative messages and summary statistics.')
+
+parser.add_argument('--verbose', '-v',
+ action="store_true",
+ help='print additional information for each processed file. '
+ 'Overwrites --quiet.')
+
+parser.add_argument('--force', '-f',
+ action="store_true",
+ help='force execution on trees with uncommited changes.')
+
+parser.add_argument('--merge', '-m',
+ action="store_true",
+ help='include merge commits. Leads to more recent mtimes '
+ 'and more files per commit, thus with the same mtime '
+ '(which may or may not be what you want). Including '
+ 'merge commits may lead to less commits being evaluated '
+ '(all files are found sooner), which improves performance, '
+ 'sometimes substantially. But since merge commits are '
+ 'usually huge, processing them may also take longer, '
+ 'sometimes substantially. By default merge logs are only '
+ 'used for files missing from regular commit logs.')
+
+parser.add_argument('--skip-missing', '-s',
+ action="store_false", default=True, dest='missing',
+ help='do not try to find missing files. If some files were '
+ 'not found in regular commit logs, by default it re-tries '
+ 'using merge commit logs for these files (if --merge was '
+ 'not used already). This option disables this behavior, '
+ 'which may slightly improve performance, but files '
+ 'found only in merge commits will not be updated.')
+
+parser.add_argument('--no-directories', '-D',
+ action="store_false", default=True, dest='dirs',
+ help='do not update directory mtime for files created, '
+ 'renamed or deleted in it. Note: just modifying a file '
+ 'will not update its directory mtime.')
+
+parser.add_argument('--test', '-t',
+ action="store_true", default=False, dest='test',
+ help='test run: do not actually update any file')
+
+parser.add_argument('pathspec',
+ nargs='*', default=[os.path.curdir],
+ help='only modify paths (dirs or files) matching PATHSPEC, '
+ 'relative to current directory. '
+ 'Default is to modify all non-ignored, tracked files.')
+
+parser.add_argument('--work-tree',
+ dest='workdir',
+ help='specify where the work tree is. '
+ 'Default for most repositories is current directory.')
+
+parser.add_argument('--git-dir',
+ dest='gitdir',
+ help='specify where the git repository is. '
+ 'Default for most repositories <work-tree>/.git')
+
+args = parser.parse_args()
+
+gitcmd = ['git']
+if args.workdir: gitcmd.append("--work-tree=%s" % args.workdir)
+if args.gitdir : gitcmd.append("--git-dir=%s" % args.gitdir)
+
+if args.verbose: level = logger.DEBUG
+elif args.quiet: level = logger.WARN
+else: level = logger.INFO
+
+logger.basicConfig(level=level, format='%(message)s')
+
+
+# UI done, it's show time!
+start = time.time() # yes, Wall time. CPU time is not realistic for users.
+
+lines = loglines = commits = totalfiles = \
+ignoredfiles = files = touches = errors = 0
+stepmissing = 100
+
+
+# First things first: Where and Who are we?
+try:
+ workdir, gitdir = text(subprocess.check_output(gitcmd + shlex.split(
+ 'rev-parse --show-toplevel --git-dir'))).split('\n')[:2]
+
+ workdir = os.path.abspath(workdir)
+ gitdir = os.path.abspath(gitdir)
+
+except subprocess.CalledProcessError as e:
+ # rev-parse couldn't find git repo, and already informed user.
+ # So we just...
+ sys.exit(e.returncode)
+
+
+# Get the files managed by git
+lsfileslist = set()
+gitobj = subprocess.Popen(gitcmd + shlex.split('ls-files --full-name') +
+ ['--'] + args.pathspec,
+ stdout=subprocess.PIPE)
+for line in gitobj.stdout:
+ lsfileslist.add(os.path.relpath(line.strip(), workdir))
+
+# List files matching user pathspec, relative to current directory
+# git commands always print paths relative to work tree root
+filelist = set()
+dirlist = set()
+for path in args.pathspec:
+
+ # Normalize user input so ./doc = doc/ = doc/../doc/. = doc
+ path = os.path.normpath(path)
+
+ # Is path inside the work tree?
+ if os.path.commonprefix([workdir, os.path.abspath(path)]) != workdir:
+ logger.warn("WARNING: Skipping pathspec outside work tree: %s", path)
+ continue
+
+ # git does not care if it's a broken symlink, hence lexists
+ if not os.path.lexists(path):
+ logger.warn("WARNING: Skipping non-existing pathspec: %s", path)
+ continue
+
+ # file or symlink (to file, to dir or broken - git handles the same way)
+ if os.path.isfile(path) or os.path.islink(path):
+ # Always add them relative to worktree root
+ filelist.add(os.path.relpath(path, workdir))
+
+ # dir
+ else:
+ for root, subdirs, files in os.walk(path):
+ if gitdir in [os.path.abspath(os.path.join(root, subdir))
+ for subdir in subdirs]:
+ subdirs.remove(os.path.basename(gitdir))
+
+ if os.path.abspath(root) == workdir and '.git' in files:
+ files.remove('.git')
+
+ if args.dirs:
+ dir = os.path.relpath(root, workdir)
+ if dir == '.':
+ dir = ''
+ dirlist.add(dir)
+
+ for file in files:
+ # Always add them relative to worktree root
+ filelist.add(os.path.relpath(os.path.join(root, file), workdir))
+
+filelist &= lsfileslist
+
+totaldirs = dirs = len(dirlist)
+totalfiles = files = len(filelist)
+logger.info("{:,} files to be processed in work dir".format(totalfiles))
+
+
+# Discard untracked and ignored files
+ignoredlist = set()
+gitobj = subprocess.Popen(gitcmd + shlex.split('status --porcelain --ignored') +
+ ['--'] + args.pathspec,
+ stdout=subprocess.PIPE)
+for line in gitobj.stdout:
+ line = text(line.strip())
+ status = line[:2]
+ filespec = line[3:]
+
+ # Make sure the slash matches the os; for Windows we need a backslash
+ if not os.sep == '/':
+ filespec = filespec.replace('/', os.sep)
+
+ if status in ['??', '!!']: # also safe to ignore: 'A ', ' M'
+ # filespec can be a dir, so we must iterate on filelist
+ for file in filelist:
+ if ( (filespec.endswith(os.sep) and file.startswith(filespec)) or
+ (file == filespec) ):
+ logger.debug("Ignoring: %s", file)
+ ignoredlist.add(file)
+ files -= 1
+ ignoredfiles += 1
+ elif not args.force:
+ logger.critical(
+ "ERROR: There are local changes in the working directory.\n"
+ "This could lead to undesirable results for modified files.\n"
+ "Please, commit your changes (or use --force) and try again.\n"
+ "Aborting")
+ gitobj.kill()
+ sys.exit(1)
+
+if ignoredfiles:
+ filelist -= ignoredlist
+ logger.info("{:,} files to process after ignoring {:,}"
+ "".format(files, ignoredfiles))
+
+
+# Process the log until all files are 'touched'
+logger.debug("Line #\tLog #\tFiles\tmtime\tFile")
+def parselog(merge=False, filterlist=[]):
+ global loglines, dirs, files, touches, errors, commits
+
+ gitobj = subprocess.Popen(gitcmd + shlex.split('whatchanged --pretty=%at') +
+ (['-m'] if merge else []) +
+ ['--'] + filterlist,
+ stdout=subprocess.PIPE)
+ for line in gitobj.stdout:
+ loglines += 1
+ line = text(line.strip())
+
+ # Blank line between Date and list of files
+ if not line: continue
+
+ # File line
+ if line.startswith(':'):
+ # If line describes a rename, linetok has three tokens, otherwise
+ # two.
+ linetok = line.split('\t')
+ status = linetok[0]
+ file = linetok[-1]
+
+ # Make sure the slash matches the os; for Windows we need a backslash
+ if not os.sep == '/':
+ file = file.replace('/',os.sep)
+
+ if file.startswith('"'):
+ file = file[1:-1].decode("string-escape")
+
+ if file in filelist:
+ logger.debug("%d\t%d\t%d\t%s\t%s",
+ loglines, commits, files,
+ time.ctime(mtime), file)
+ filelist.remove(file)
+ files -= 1
+ try:
+ if not args.test:
+ os.utime(os.path.join(workdir, file), (mtime, mtime))
+ touches += 1
+ except Exception as e:
+ logger.error("ERROR: %s\n", e)
+ errors += 1
+
+ if args.dirs:
+ dir = os.path.dirname(file)
+ if status[-1] in ['A', 'D'] and dir in dirlist:
+ logger.debug("%d\t%d\t-\t%s\t%s",
+ loglines, commits,
+ time.ctime(mtime), dir)
+ dirlist.remove(dir)
+ try:
+ if not args.test:
+ os.utime(os.path.join(workdir, dir), (mtime, mtime))
+ dirs -= 1
+ except Exception as e:
+ logger.error("ERROR: %s\n", e)
+
+ # Date line
+ else:
+ commits += 1
+ mtime = int(line)
+
+ # All files done?
+ if not files:
+ break
+
+ try:
+ gitobj.terminate()
+ except OSError:
+ pass
+
+
+parselog(args.merge, args.pathspec)
+
+# Missing files
+if filelist:
+
+ # Try to find them in merge logs, if not done already
+ # (usually HUGE, thus MUCH slower!)
+ if args.missing and not args.merge:
+ filterlist = list(filelist)
+ for i in range(0, len(filterlist), stepmissing):
+ parselog(merge=True, filterlist=filterlist[i:i+stepmissing])
+
+ # Still missing some?
+ for file in filelist:
+ logger.warn("WARNING: not found in log: %s", file)
+
+
+# Final statistics
+# TODO: use git-log --before=mtime to brag about skipped log entries
+logger.info(
+ "Statistics:\n"
+ "{:13,.2f} seconds\n"
+ "{:13,} log lines processed\n"
+ "{:13,} commits evaluated"
+ "".format(time.time()-start, loglines, commits))
+
+if touches != totalfiles:
+ logger.info("{:13,} total files".format(totalfiles))
+if ignoredfiles: logger.info("{:13,} ignored files".format(ignoredfiles))
+if files: logger.info("{:13,} missing files".format(files))
+if errors: logger.info("{:13,} update errors".format(errors))
+
+logger.info("{:13,} updated files".format(touches))
+
+if args.dirs:
+ logger.info("{:13,} updated directories".format(totaldirs - dirs))
+
+if args.test:
+ logger.info("TEST RUN - No files modified!")
+
+# Statistics for some large projects
+
+#bash
+# 0.27 seconds
+# 5,750 log lines processed
+# 62 commits evaluated
+# 1,155 updated files
+
+#git
+# 3.71 seconds
+# 96,702 log lines processed
+# 24,217 commits evaluated
+# 2,495 updated files
+
+#git (--merge)
+# 0.19 seconds
+# 2,586 log lines processed
+# 3 commits evaluated
+# 2,495 updated files
+
+#wine
+# 13.53 seconds
+# 443,979 log lines processed
+# 91,703 commits evaluated
+# 6,005 updated files
+
+#linux kernel
+# 59.11 seconds
+# 1,484,567 log lines processed
+# 313,164 commits evaluated
+# 40,902 updated files
+
+#linux kernel (--skip-missing)
+#WARNING: not found in log: arch/arm/mach-sa1100/include/mach/reset.h
+#WARNING: not found in log: lib/raid6/unroll.awk
+#Statistics:
+# 51.88 seconds
+# 1,484,558 log lines processed
+# 313,161 commits evaluated
+# 40,902 total files
+# 2 missing files
+# 40,900 updated files
+
+#linux kernel (--merge)
+# 501.17 seconds
+# 34,495,300 log lines processed
+# 238,711 commits evaluated
+# 40,902 updated files
diff --git a/mastermirror-staging.sh b/mastermirror-staging.sh
index 3952c4f..4440ecb 100755
--- a/mastermirror-staging.sh
+++ b/mastermirror-staging.sh
@@ -46,20 +46,24 @@ fi
if [[ ! -d ${STAGING_DIR}/gentoo-x86 ]] ; then
# not checked out yet, run initial co
${GIT_CL} file://${GITROOT}/repo/gentoo.git/ ${STAGING_DIR}/gentoo-x86
+ cd ${STAGING_DIR}/gentoo-x86
else
cd ${STAGING_DIR}/gentoo-x86
${GIT_PL}
fi
+/usr/local/bin/mastermirror/git-restore-mtime -m -q
# gentoo-news
if [[ ! -d ${STAGING_DIR}/gentoo-news ]]; then
# not checked out yet, run initial co
cd ${STAGING_DIR}
${GIT_CL} file://${GITROOT}/data/gentoo-news.git/ gentoo-news
+ cd ${STAGING_DIR}/gentoo-news
else
cd ${STAGING_DIR}/gentoo-news
${GIT_PL}
fi
+/usr/local/bin/mastermirror/git-restore-mtime -m -q
# herds.xml
if [[ ! -d ${STAGING_DIR}/herds ]]; then
@@ -82,26 +86,32 @@ if [[ ! -d ${STAGING_DIR}/dtd ]]; then
# not checked out yet, run initial co
cd ${STAGING_DIR}
${GIT_CL} file://${GITROOT}/data/dtd.git/ dtd
+ cd ${STAGING_DIR}/dtd
else
cd ${STAGING_DIR}/dtd
${GIT_PL}
fi
+/usr/local/bin/mastermirror/git-restore-mtime -m -q
# glsa
if [[ ! -d ${STAGING_DIR}/glsa ]]; then
# not checked out yet, run initial co
cd ${STAGING_DIR}
${GIT_CL} file://${GITROOT}/data/glsa.git/ glsa
+ cd ${STAGING_DIR}/glsa
else
cd ${STAGING_DIR}/glsa
${GIT_PL}
fi
+/usr/local/bin/mastermirror/git-restore-mtime -m -q
# changelogs
if [[ ! -d ${STAGING_CHANGELOG_DIR} ]]; then
# not checked out yet, run initial co
${GIT_CL} file://${GITROOT}/data/gentoo-changelogs.git/ ${STAGING_CHANGELOG_DIR}
+ cd ${STAGING_CHANGELOG_DIR}
else
cd ${STAGING_CHANGELOG_DIR}
${GIT_PL}
fi
+/usr/local/bin/mastermirror/git-restore-mtime -m -q
diff --git a/rsync-gen.sh b/rsync-gen.sh
index 048e196..3023351 100755
--- a/rsync-gen.sh
+++ b/rsync-gen.sh
@@ -14,6 +14,13 @@
# 6) place herds.xml and projects.xml in STAGEDIR
# 7) rsync from STAGEDIR to FINALDIR
+################################################################################
+# NOTE: You MUST run git-restore-mtime -m on the git checkouts hosted in
+# git.gentoo.org::mastermirror-staging/ or this WILL NOT WORK.
+# You can get git-restore-mtime here:
+# https://github.com/MestreLion/git-tools/blob/master/git-restore-mtime
+################################################################################
+
umask 022
source /usr/local/bin/mastermirror/rsync-gen.vars
@@ -73,6 +80,18 @@ ${RSYNC} ${RSYNC_ARGS} ${RSYNC_ARGS_DELETE} git.gentoo.org::mastermirror-staging
timelog___end "STAGING RSYNC" | timelogger
# end 1)
+# start 1a)
+# Temporary: import old CVS changelogs
+# This will cause some churn, because it'll copy changelogs for packages which
+# don't exist anymore, and then the next rsync will delete them. You can reduce
+# churn by occasionally culling dead packages from the repo
+timelog_start "CHANGELOG-IMPORT" | timelogger
+rsync -Wqa \
+ $RSYNC_GIT_EXCLUDE \
+ ${EXPORTS}/changelogs/ ${STAGEDIR}
+timelog___end "CHANGELOG-IMPORT" | timelogger
+# end 1a)
+
# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR
# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR
@@ -82,12 +101,12 @@ timelog___end "STAGING RSYNC" | timelogger
# Keep /metadata/cache around so the --rsync switch of egencache will work as
# designed
timelog_start "STAGEDIR RSYNC" | timelogger
-rsync -Wqa \
+rsync -Wqau \
$RSYNC_GIT_EXCLUDE \
- --exclude=/metadata/timestamp.x \
- --exclude=/metadata/md5-cache \
- --exclude=ChangeLog \
- --delete \
+ --filter 'P /metadata/***' \
+ --filter 'Pp Manifest' \
+ --filter 'Pp ChangeLog*' \
+ --delete --delete-excluded \
${EXPORTS}/gentoo-x86/ ${STAGEDIR}/ || exit
timelog___end "STAGEDIR RSYNC" | timelogger
# end 1b)
@@ -100,20 +119,6 @@ done
timelog___end "ECLASS CHECK" | timelogger
# end 1c)
-# start 1d)
-# Temporary: import old CVS changelogs
-timelog_start "CHANGELOG-IMPORT" | timelogger
-for file in $EXPORTS/changelogs/*/*/ChangeLog*; do
- # remove the prefix so we can copy
- file=${file#$EXPORTS/changelogs/}
- dir=$(dirname "$file")
- # do the copy, the stderr redirect is for changelogs that no longer have an ebuild
- # (which is why rsync is not used, as it would create a directory/package when none should exist)
- [ -d "${STAGEDIR}/${dir}" ] && cp -a $EXPORTS/changelogs/${file} ${STAGEDIR}/${file}
-done
-timelog___end "CHANGELOG-IMPORT" | timelogger
-# end 1d)
-
# 2) generate metadata (egencache)
# ${STAGEDIR}/metadata/cache is created automatically
export PORTAGE_USERNAME=gmirror PORTAGE_GRPNAME=gmirror
@@ -148,15 +153,12 @@ function parallel_repoman_manifest_REPOMAN() {
timelog_start "REGEN" | timelogger
-# Force Manifests to thick, unsigned
-# also disable commit signing for now.
-# TODO: add infra signing of Manifests
+# Ensure layout.conf is set to thin-manifests so egencache doesn't error
sed -i \
- -e '/^thin-manifests/s,true,false,g' \
+ -e '/^thin-manifests/s,false,true,g' \
-e '/^sign-manifests/s,true,false,g' \
- -e '/^sign-commits/s,true,false,g' \
+ -e '/^sign-commits/s,false,true,g' \
${STAGEDIR}/metadata/layout.conf
-parallel_repoman_manifest_REPOMAN
# Only update the changelogs every 6 hours
# because right now it takes a very long time to do it (exceeding 1 hour at times).
@@ -171,9 +173,11 @@ esac
GIT_DIR=${EXPORTS}/gentoo-x86/.git/ \
egencache --update --rsync $PARALLEL_PARAMS \
--tolerant --cache-dir=${BASE}/tmp/ \
- --portdir=${STAGEDIR} \
+ --repositories-configuration="
+[gentoo]
+location = ${STAGEDIR}
+" \
--update-use-local-desc \
- --update-manifests --thin-manifests=n \
--repo=gentoo \
$EGENCACHE_CHANGELOG \
>> ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} 2>&1
@@ -187,10 +191,6 @@ if [[ ${rval} != 0 ]]; then
exit 5
fi
-# Redo the Manifests, because the ChangeLog is not added to the Manifest for
-# some reason.
-parallel_repoman_manifest_REPOMAN
-
# don't save empty files
if [[ ! -s ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} ]]; then
rm ${REGEN_LOG_DIR}/${REGEN_LOG_FILE}
@@ -199,6 +199,20 @@ fi
# Keep 30 days of logs
find ${REGEN_LOG_DIR} -type f -mtime +30 -print0 | xargs -0r rm
+# Force Manifests to thick, unsigned
+# also disable commit signing for now.
+# TODO: add infra signing of Manifests
+sed -i \
+ -e '/^thin-manifests/s,true,false,g' \
+ -e '/^sign-manifests/s,true,false,g' \
+ -e '/^sign-commits/s,true,false,g' \
+ ${STAGEDIR}/metadata/layout.conf
+
+# Thicken manifests
+timelog_start "THICKEN" | timelogger
+parallel_repoman_manifest_THICKEN
+timelog___end "THICKEN" | timelogger
+
# Mark that metadata is done
date -u > ${STAGEDIR}/metadata/timestamp
diff --git a/thicken-manifests.py b/thicken-manifests.py
index 3b04fa4..db52bdc 100755
--- a/thicken-manifests.py
+++ b/thicken-manifests.py
@@ -8,10 +8,10 @@
# alright too)
# - Run egencache against the checkout (this step is NOT optional; manifest generation is torturously slow without md5-cache)
# - Run thicken-manifests.py
-#
-# Takes an optional --jobs parameter, defaults to the number of CPUs on the host; also takes an optional directory to run against, defaults to the current working directory.
-# For GPG signing, add --sign parameter.
-# It behaves exactly like repoman manifest, so you can override GPG behavior by setting PORTAGE_GPG_SIGNING_COMMAND, PORTAGE_GPG_KEY, and PORTAGE_GPG_DIR in make.conf or the environment.
+#
+# Takes an optional --jobs parameter, defaults to the number of CPUs on the host; also takes an optional directory to run against, defaults to the current working directory
+# For GPG signing, add --sign parameter.
+# It behaves exactly like repoman manifest, so you can override GPG behavior by setting PORTAGE_GPG_SIGNING_COMMAND, PORTAGE_GPG_KEY, and PORTAGE_GPG_DIR in make.conf or the environment.
# You do NOT need to modify layout.conf for this script to work. Stick with GPG 2.0 for now, because 2.1 moves key operations into the agent, which makes them not parallel.
import os
@@ -21,6 +21,7 @@ import logging
import argparse
import errno
import math
+import collections
import portage
import portage.exception
@@ -31,6 +32,7 @@ portage_settings = None
portage_portdbapi = None
logger = None
args = None
+dir_contents = collections.defaultdict(list)
def gpg_sign(filename):
@@ -79,8 +81,9 @@ def maybe_thicken_manifest(pkg_dir):
logger.exception("%s OSError thrown trying to stat Manifest", pkg_dir)
manifest_mtime = 0
newest_mtime = manifest_mtime
- for root, subdirectories, files in os.walk(pkg_dir):
- for file in files:
+ manifest_entries = []
+ for root in pkg_dir, os.path.join(pkg_dir, 'files'):
+ for file in dir_contents[root]:
if file == 'Manifest':
continue
@@ -88,11 +91,29 @@ def maybe_thicken_manifest(pkg_dir):
if file_mtime > newest_mtime:
newest_mtime = file_mtime
+ if os.path.split(root)[1] == 'files':
+ manifest_entries.append('AUX ' + file)
+ elif file.endswith('.ebuild'):
+ manifest_entries.append('EBUILD ' + file)
+ else:
+ manifest_entries.append('MISC ' + file)
+
if newest_mtime == manifest_mtime:
try:
with open(os.path.join(pkg_dir, 'Manifest'), 'r') as f:
- if f.readline().startswith('-'):
- return
+ for line in f:
+ if not line.startswith(('AUX', 'EBUILD', 'MISC')):
+ continue
+
+ manifest_entry = ' '.join(line.split(' ')[0:2])
+ if manifest_entry in manifest_entries:
+ manifest_entries.remove(manifest_entry)
+ else:
+ newest_mtime += 1
+ break
+ else:
+ if not manifest_entries:
+ return
except:
pass
@@ -112,13 +133,14 @@ def maybe_thicken_manifest(pkg_dir):
def main():
- global logger, args
+ global logger, args, dir_contents
parser = argparse.ArgumentParser(description='Thicken ebuild manifests as needed')
parser.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int,
- help='Number of parallel jobs to run; default: number of CPUs')
- parser.add_argument('-s', '--sign', action='store_true')
+ help='Number of parallel jobs to run; default: number of CPUs')
+ parser.add_argument('-s', '--sign', action='store_true', help='Sign manifests with GPG')
parser.add_argument('location', nargs='?', default='.', help='The location to thicken manifests in; default: .')
args = parser.parse_args()
+ args.location = os.path.realpath(args.location)
logger = multiprocessing.log_to_stderr()
logger.setLevel(logging.WARN)
@@ -129,18 +151,31 @@ main-repo = gentoo
[gentoo]
location = %s
-''' % os.path.realpath(args.location)
+''' % args.location
os.chdir(args.location)
pkg_dirs = []
for root, subdirectories, files in os.walk('.'):
- for dir in ('eclass', 'metadata', 'profiles', '.git', 'files'):
- if dir in subdirectories:
+ if root == '.':
+ for dir in ('eclass', 'licenses', 'metadata', 'profiles', 'scripts', '.git'):
+ if dir in subdirectories:
+ subdirectories.remove(dir)
+
+ if (not subdirectories or subdirectories == ['files']) and 'metadata.xml' in files:
+ pkg_dirs.append(root[2:])
+ dir_contents[root[2:]] = files
+ elif os.path.split(root)[1] == 'files' and 'metadata.xml' not in files:
+ dir_contents[root[2:]] = files
+ for dir in subdirectories[:]:
subdirectories.remove(dir)
-
- if not subdirectories and 'metadata.xml' in files:
- pkg_dirs.append(root)
-
+ os.chdir(root)
+ for root2, subdirectories2, files2 in os.walk('.'):
+ if root2 != '.':
+ for file in files2:
+ dir_contents[root[2:]].append(os.path.join(root2[2:], file))
+ os.chdir(args.location)
+
+ pkg_dirs.sort()
pool = multiprocessing.Pool(args.jobs, worker_init)
pool.map(maybe_thicken_manifest, pkg_dirs)
pool.close()
|
Comments (0)
You can clone a snippet to your computer for local editing. Learn more.