Rearranged and commented files for running under Jenkins.

.travis.yml

@@ -10,7 +10,7 @@ os:
 # The Objective C build needs Xcode 7.0 or later.
 osx_image: xcode7.2
 script:
-  - ./tools/run_tests/tests.sh $CONFIG
+  - ./tests.sh $CONFIG
 env:
   - CONFIG=cpp
   - CONFIG=cpp_distcheck

jenkins/README.md

@@ -0,0 +1,6 @@
+
+Jenkins Infrastructure
+----------------------
+
+The scripts in this directory serve as plumbing for running the protobuf
+tests under Jenkins.

jenkins/build_and_run_docker.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Builds docker image and runs a command under it.
+# This is a generic script that is configured with the following variables:
+#
+# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
+# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
+# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
+# $@ - Extra args to pass to docker run
+
+
+set -ex
+
+cd $(dirname $0)/..
+git_root=$(pwd)
+cd -
+
+# Use image name based on Dockerfile location checksum
+DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
+
+# Make sure docker image has been built. Should be instantaneous if so.
+docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
+
+# Ensure existence of ccache directory
+CCACHE_DIR=/tmp/protobuf-ccache
+mkdir -p $CCACHE_DIR
+
+# Choose random name for docker container
+CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
+
+# Run command inside docker
+docker run \
+  "$@" \
+  -e CCACHE_DIR=$CCACHE_DIR \
+  -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \
+  -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+  -v "$git_root:/var/local/jenkins/protobuf:ro" \
+  -v $CCACHE_DIR:$CCACHE_DIR \
+  -w /var/local/git/protobuf \
+  --name=$CONTAINER_NAME \
+  $DOCKER_IMAGE_NAME \
+  bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
+
+# Copy output artifacts
+if [ "$OUTPUT_DIR" != "" ]
+then
+  docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true"
+fi
+
+# remove the container, possibly killing it first
+docker rm -f $CONTAINER_NAME || true
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

jenkins/buildcmds/README.md

@@ -0,0 +1,6 @@
+
+Jenkins Build Commands
+----------------------
+
+The scripts in this directory are designed to be top-level entry points for
+Jenkins projects.

jenkins/buildcmds/pull_request.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Jenkins as the entry point for
+# running the "pull request" project:
+#
+#   https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+export DOCKERFILE_DIR=jenkins/docker
+export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+./jenkins/build_and_run_docker.sh

tools/docker/Dockerfile → jenkins/docker/Dockerfile

@@ -1,31 +1,10 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
+# This Dockerfile specifies the recipe for creating an image for the tests
+# to run in.
 #
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# We install as many test dependencies here as we can, because these setup
+# steps can be cached.  They do *not* run every time we run the build.
+# The Docker image is only rebuilt when the Dockerfile (ie. this file)
+# changes.
 
 # Base Dockerfile for gRPC dev images
 FROM debian:latest

tools/jenkins/make_test_output.py → jenkins/make_test_output.py

@@ -1,9 +1,9 @@
-"""Gather output from test runs and create an XML file in JUnit format.
+"""Gathers output from test runs and create an XML file in JUnit format.
 
 The output files from the individual tests have been written in a directory
 structure like:
 
-  $DIR/joblog  (--joblog from "parallel")
+  $DIR/joblog  (output from "parallel --joblog joblog")
   $DIR/logs/1/cpp/stdout
   $DIR/logs/1/cpp/stderr
   $DIR/logs/1/csharp/stdout
@@ -13,7 +13,8 @@ structure like:
   etc.
 
 This script bundles them into a single output XML file so Jenkins can show
-detailed test results.
+detailed test results.  It runs as the last step before the Jenkins build
+finishes.
 """
 
 import os;

tools/run_tests/jenkins.sh → jenkins/pull_request_in_docker.sh

@@ -1,22 +1,25 @@
 #!/bin/bash
+#
+# This is the script that runs inside Docker, once the image has been built,
+# to execute all tests for the "pull request" project.
 
 WORKSPACE_BASE=`pwd`
 MY_DIR="$(dirname "$0")"
-TEST_SCRIPT=$MY_DIR/tests.sh
+TEST_SCRIPT=$MY_DIR/../tests.sh
 BUILD_DIR=/tmp/protobuf
 
-# Set value used in tests.sh.
-PARALLELISM=-j8
-
 set -e  # exit immediately on error
 set -x  # display all commands
 
+# The protobuf repository is mounted into our Docker image, but read-only.
+# We clone into a directory inside Docker (this is faster than cp).
 rm -rf $BUILD_DIR
 mkdir -p $BUILD_DIR
 cd $BUILD_DIR
 git clone /var/local/jenkins/protobuf
 cd protobuf
 
+# Set up the directory where our test output is going to go.
 OUTPUT_DIR=`mktemp -d`
 LOG_OUTPUT_DIR=$OUTPUT_DIR/logs
 mkdir -p $LOG_OUTPUT_DIR/1/cpp
@@ -35,14 +38,14 @@ mkdir -p $LOG_OUTPUT_DIR/1/cpp
 CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout
 CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr
 
+# Time the C++ build, so we can put this info in the test output.
 # It's important that we get /usr/bin/time (which supports -f and -o) and not
 # the bash builtin "time" which doesn't.
 TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time"
 
 $TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
 
-# Other tests are run in parallel.  The overall run fails if any one of them
-# fails.
+# Other tests are run in parallel.
 
 parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
   csharp \
@@ -55,14 +58,15 @@ parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT :::
   ruby21 \
   || true  # Process test results even if tests fail.
 
+cat $OUTPUT_DIR/joblog
+
 # The directory that is copied from Docker back into the Jenkins workspace.
 COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput
 mkdir -p $COPY_FROM_DOCKER
 TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml
 
-python $MY_DIR/../jenkins/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
+# Process all the output files from "parallel" and package them into a single
+# .xml file with detailed, broken-down test output.
+python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
 
 ls -l $TESTOUTPUT_XML_FILE
-
-### disabled tests
-# java_jdk6 \

tools/run_tests/tests.sh → tests.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+#
+# Build and runs tests for the protobuf project.  The tests as written here are
+# used by both Jenkins and Travis, though some specialized logic is required to
+# handle the differences between them.
 
 on_travis() {
   if [ "$TRAVIS" == "true" ]; then
@@ -25,19 +29,19 @@ internal_build_cpp() {
 
   ./autogen.sh
   ./configure
-  make $PARALLELISM
+  make -j2
 }
 
 build_cpp() {
   internal_build_cpp
-  make check $PARALLELISM
+  make check -j2
   cd conformance && make test_cpp && cd ..
 }
 
 build_cpp_distcheck() {
   ./autogen.sh
   ./configure
-  make distcheck $PARALLELISM
+  make distcheck -j2
 }
 
 build_csharp() {
@@ -308,8 +312,6 @@ build_javascript() {
   cd js && npm install && npm test && cd ..
 }
 
-[ -n "${PARALLELISM}" ] && PARALLELISM=-j8
-
 # Note: travis currently does not support testing more than one language so the
 # .travis.yml cheats and claims to only be cpp.  If they add multiple language
 # support, this should probably get updated to install steps and/or
@@ -320,9 +322,6 @@ build_javascript() {
 
 # -------- main --------
 
-# Set value used in tests.sh.
-PARALLELISM=-j2
-
 if [ "$#" -ne 1 ]; then
   echo "
 Usage: $0 { cpp |

tools/jenkins/build_and_run_docker.sh

@@ -1,84 +0,0 @@
-#!/bin/bash
-# Copyright 2016, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# Builds docker image and runs a command under it.
-# You should never need to call this script on your own.
-
-set -ex
-
-cd $(dirname $0)/../..
-git_root=$(pwd)
-cd -
-
-# Inputs
-# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
-# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
-# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
-# $@ - Extra args to pass to docker run
-
-# Use image name based on Dockerfile location checksum
-DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
-
-# Make sure docker image has been built. Should be instantaneous if so.
-docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
-
-# Ensure existence of ccache directory
-CCACHE_DIR=/tmp/protobuf-ccache
-mkdir -p $CCACHE_DIR
-
-# Choose random name for docker container
-CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
-
-# Run command inside docker
-docker run \
-  "$@" \
-  -e CCACHE_DIR=$CCACHE_DIR \
-  -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \
-  -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
-  -v "$git_root:/var/local/jenkins/protobuf:ro" \
-  -v $CCACHE_DIR:$CCACHE_DIR \
-  -w /var/local/git/protobuf \
-  --name=$CONTAINER_NAME \
-  $DOCKER_IMAGE_NAME \
-  bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
-
-# Copy output artifacts
-if [ "$OUTPUT_DIR" != "" ]
-then
-  docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true"
-fi
-
-# remove the container, possibly killing it first
-docker rm -f $CONTAINER_NAME || true
-
-if [ "$FAILED" != "" ]
-then
-  exit 1
-fi

tools/jenkins/pull_request.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-
-export DOCKERFILE_DIR=tools/docker
-export DOCKER_RUN_SCRIPT=tools/run_tests/jenkins.sh
-export OUTPUT_DIR=testoutput
-./tools/jenkins/build_and_run_docker.sh