Commits

Peter Eisentraut committed 6b7dc14

Add linkcheck job

This should eventually replace the linklint job, which has turned out to
be too unstable and inflexible.

Comments (0)

Files changed (6)

jenkins/jobs/postgresql_master_linkcheck/config.xml

+<?xml version='1.0' encoding='UTF-8'?>
+<project>
+  <actions/>
+  <description></description>
+  <keepDependencies>false</keepDependencies>
+  <properties>
+    <hudson.plugins.buildblocker.BuildBlockerProperty plugin="build-blocker-plugin@1.2">
+      <useBuildBlocker>false</useBuildBlocker>
+    </hudson.plugins.buildblocker.BuildBlockerProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <quietPeriod>300</quietPeriod>
+  <canRoam>true</canRoam>
+  <disabled>false</disabled>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <triggers class="vector"/>
+  <concurrentBuild>false</concurrentBuild>
+  <builders>
+    <hudson.plugins.copyartifact.CopyArtifact plugin="copyartifact@1.25">
+      <projectName>postgresql_master_world</projectName>
+      <filter>doc/src/sgml/postgres.xml</filter>
+      <target></target>
+      <selector class="hudson.plugins.copyartifact.StatusBuildSelector"/>
+      <flatten>true</flatten>
+    </hudson.plugins.copyartifact.CopyArtifact>
+    <hudson.tasks.Shell>
+      <command>bash /srv/pgci/jobhelpers/checklinks.sh postgres.xml</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers>
+    <org.tap4j.plugin.TapPublisher plugin="tap@1.9">
+      <testResults>checklinks.tap</testResults>
+      <failIfNoResults>true</failIfNoResults>
+      <failedTestsMarkBuildAsFailure>false</failedTestsMarkBuildAsFailure>
+      <outputTapToConsole>true</outputTapToConsole>
+      <enableSubtests>false</enableSubtests>
+      <discardOldReports>false</discardOldReports>
+      <todoIsFailure>false</todoIsFailure>
+    </org.tap4j.plugin.TapPublisher>
+  </publishers>
+  <buildWrappers>
+    <hudson.plugins.ws__cleanup.PreBuildCleanup plugin="ws-cleanup@0.10">
+      <deleteDirs>false</deleteDirs>
+    </hudson.plugins.ws__cleanup.PreBuildCleanup>
+    <hudson.plugins.timestamper.TimestamperBuildWrapper plugin="timestamper@1.3.2"/>
+  </buildWrappers>
+</project>

jenkins/jobs/postgresql_master_world/config.xml

   <actions/>
   <description></description>
   <keepDependencies>false</keepDependencies>
-  <properties/>
+  <properties>
+    <hudson.plugins.buildblocker.BuildBlockerProperty plugin="build-blocker-plugin@1.2">
+      <useBuildBlocker>false</useBuildBlocker>
+    </hudson.plugins.buildblocker.BuildBlockerProperty>
+  </properties>
   <scm class="hudson.plugins.git.GitSCM" plugin="git@1.1.26">
     <configVersion>2</configVersion>
     <userRemoteConfigs>
       </consoleParsers>
     </hudson.plugins.warnings.WarningsPublisher>
     <hudson.tasks.ArtifactArchiver>
-      <artifacts>**/regression.diffs,**/regression.out,postgresql-*.tar.*</artifacts>
+      <artifacts>**/regression.diffs,**/regression.out,postgresql-*.tar.*,doc/src/sgml/postgres.xml</artifacts>
       <latestOnly>false</latestOnly>
     </hudson.tasks.ArtifactArchiver>
     <hudson.plugins.textfinder.TextFinderPublisher plugin="text-finder@1.9">
       <recordBuildArtifacts>false</recordBuildArtifacts>
     </hudson.tasks.Fingerprinter>
     <hudson.tasks.BuildTrigger>
-      <childProjects>postgresql_master_coverage, postgresql_master_dist, postgresql_master_extradoc, plproxy, pgvihash, pg_logforward, pghashlib, postgresql_master_linklint</childProjects>
+      <childProjects>postgresql_master_coverage, postgresql_master_dist, postgresql_master_extradoc, plproxy, pgvihash, pg_logforward, pghashlib, postgresql_master_linklint, postgresql_master_linkcheck</childProjects>
       <threshold>
         <name>UNSTABLE</name>
         <ordinal>1</ordinal>

jobhelpers/checklinks.sh

+#!/bin/bash
+
+set -eux
+set -o pipefail
+
+mydir=$(dirname $0)
+
+flaky_hosts='db.cs.berkeley.edu zlatkovic.com'
+
+urls=$(xsltproc "$mydir/docbook-ulink-urls.xsl" "$1" | LC_COLLATE=C sort -u)
+
+total=0
+for url in $urls; do
+	total=$(($total + 1))
+done
+
+(
+echo "1..$total"
+
+i=0
+last_host=
+for url in $urls; do
+	i=$(($i + 1))
+
+	host=$(echo "$url" | sed -r 's,^.*://([^/]+).*$,\1,')
+
+	for h in $flaky_hosts; do
+		if [ "$host" = "$h" ]; then
+			echo "ok $i $url  # skip: flaky host"
+			continue 2
+		fi
+	done
+
+	# If checking a URL from the same host as the last one, insert
+	# a longer delay.  Some hosts apparently don't like clients
+	# reconnecting quickly.
+	if [ "$host" == "$last_host" ]; then
+		sleep 1
+	fi
+
+	if out=$(curl --fail --location --max-time 30 --retry 5 --silent --show-error --output /dev/null "$url" 2>&1); then
+		echo "ok $i $url"
+	else
+		echo "not ok $i $url"
+		echo "$out" | sed 's/^/    # /'
+	fi
+
+	sleep 0.1
+	last_host=$host
+done
+) >checklinks.tap

manifests/pgci.pp

 jenkins::plugin { 'git': }
 jenkins::plugin { 'googleanalytics': }
 jenkins::plugin { 'htmlpublisher': }
+jenkins::plugin { 'tap': }
 jenkins::plugin { 'text-finder': }
 jenkins::plugin { 'timestamper': }
 jenkins::plugin { 'thinBackup': }

modules/pgci-build-deps/manifests/init.pp

                   'texinfo',
                   'xsltproc',
 
+                  'curl',
                   'python-docutils', # for pghashlib
                   'time',
                   ]