Automated testing tweaks for ObjC

- Move the ObjC tests into the list and exclude them on linux, this will change
  where in the order they start, since they are longer, it will have other
  things run in parallel instead of them ending up last and taking the longest.
- Switch to the Xcode 7.3 image.
- Drop the use of xctool and stream line things through the full_mac_build.sh
  script. This means we end up with only one build script instead of two.
- Tweaks to the mac build script:
  - Make iOS Xcode version support explicit
  - Support Debug/Release only building
  - Change the OS X min parallel count to 2 to better deal with VMs.
- Split the travis ios tests into the two Xcode Configurations as the logs are
  choking travis.

.travis.yml

@@ -8,7 +8,7 @@ os:
   - linux
   - osx
 # The Objective C build needs Xcode 7.0 or later.
-osx_image: xcode7.2
+osx_image: xcode7.3
 script:
   - ./tests.sh $CONFIG
 env:
@@ -23,6 +23,11 @@ env:
   - CONFIG=javanano_jdk7
   - CONFIG=javanano_oracle7
   - CONFIG=javascript
+  # iOS build log was starting to choke travis UI, so split to cover the
+  # Xcode Debug and Release Configurations independently.
+  - CONFIG=objectivec_ios_debug
+  - CONFIG=objectivec_ios_release
+  - CONFIG=objectivec_osx
   - CONFIG=python
   - CONFIG=python_cpp
   - CONFIG=ruby19
@@ -54,12 +59,13 @@ matrix:
     # which doesn't work on OS X.
     - os: osx
       env: CONFIG=golang
-  # Add into the matrix OS X tests of Objective C (needs Xcode, so it won't
-  # work on other platforms). These are split so it doesn't take as long to run.
-  include:
-    - os: osx
-      env: CONFIG=objectivec_ios
-    - os: osx
+    # OS X/iOS tests of Objective C (needs Xcode, so it won't work on other
+    # platforms).
+    - os: linux
+      env: CONFIG=objectivec_ios_debug
+    - os: linux
+      env: CONFIG=objectivec_ios_release
+    - os: linux
       env: CONFIG=objectivec_osx
   allow_failures:
     # These currently do not work on OS X but are being worked on by @haberman.
@@ -71,14 +77,11 @@ matrix:
     # we moved to an OS X image that is 10.11.
     - os: osx
       env: CONFIG=python_cpp
-    # xctool 0.2.8 seems to have a bug where it randomly kills tests saying
-    # they failed.
-    #   https://github.com/facebook/xctool/issues/619
-    #   https://github.com/google/protobuf/issues/1232
-    # travis updated their images to include 0.2.8:
-    #   https://blog.travis-ci.com/2016-03-23-xcode-image-updates
-    # Mark the iOS test as flakey so these failures don't turn things red.
+    # Mark the iOS test as flakey as xcodebuild some times fails to start the
+    # iOS Simulator.
+    - os: osx
+      env: CONFIG=objectivec_ios_debug
     - os: osx
-      env: CONFIG=objectivec_ios
+      env: CONFIG=objectivec_ios_release
 notifications:
   email: false

objectivec/DevTools/full_mac_build.sh

@@ -37,6 +37,10 @@ OPTIONS:
          Skip the invoke of Xcode to test the runtime on both iOS and OS X.
    --skip-xcode-ios
          Skip the invoke of Xcode to test the runtime on iOS.
+   --skip-xcode-debug
+         Skip the Xcode Debug configuration.
+   --skip-xcode-release
+         Skip the Xcode Release configuration.
    --skip-xcode-osx
          Skip the invoke of Xcode to test the runtime on OS X.
    --skip-objc-conformance
@@ -66,8 +70,8 @@ wrapped_make() {
 }
 
 NUM_MAKE_JOBS=$(/usr/sbin/sysctl -n hw.ncpu)
-if [[ "${NUM_MAKE_JOBS}" -lt 4 ]] ; then
-  NUM_MAKE_JOBS=4
+if [[ "${NUM_MAKE_JOBS}" -lt 2 ]] ; then
+  NUM_MAKE_JOBS=2
 fi
 
 DO_AUTOGEN=no
@@ -76,6 +80,8 @@ REGEN_DESCRIPTORS=no
 CORE_ONLY=no
 DO_XCODE_IOS_TESTS=yes
 DO_XCODE_OSX_TESTS=yes
+DO_XCODE_DEBUG=yes
+DO_XCODE_RELEASE=yes
 DO_OBJC_CONFORMANCE_TESTS=yes
 while [[ $# != 0 ]]; do
   case "${1}" in
@@ -109,6 +115,12 @@ while [[ $# != 0 ]]; do
     --skip-xcode-osx )
       DO_XCODE_OSX_TESTS=no
       ;;
+    --skip-xcode-debug )
+      DO_XCODE_DEBUG=no
+      ;;
+    --skip-xcode-release )
+      DO_XCODE_RELEASE=no
+      ;;
     --skip-objc-conformance )
       DO_OBJC_CONFORMANCE_TESTS=no
       ;;
@@ -151,8 +163,12 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then
         -project objectivec/ProtocolBuffers_iOS.xcodeproj
         -scheme ProtocolBuffers
     )
-  "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Debug clean
-  "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Release clean
+    if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
+      "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Debug clean
+    fi
+    if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
+      "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Release clean
+    fi
   fi
   if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then
     XCODEBUILD_CLEAN_BASE_OSX=(
@@ -160,8 +176,12 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then
         -project objectivec/ProtocolBuffers_OSX.xcodeproj
         -scheme ProtocolBuffers
     )
-  "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Debug clean
-  "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Release clean
+    if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
+      "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Debug clean
+    fi
+    if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
+      "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Release clean
+    fi
   fi
 fi
 
@@ -222,20 +242,20 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then
           -destination "platform=iOS Simulator,name=iPad Air,OS=9.0" # 64bit
       )
       ;;
-    7.3* )
+    7.2* )
       XCODEBUILD_TEST_BASE_IOS+=(
           -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.3" # 64bit
+          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
           -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPad Air,OS=9.3" # 64bit
+          -destination "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
       )
       ;;
-    7.* )
+    7.3* )
       XCODEBUILD_TEST_BASE_IOS+=(
           -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
+          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.3" # 64bit
           -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
+          -destination "platform=iOS Simulator,name=iPad Air,OS=9.3" # 64bit
       )
       ;;
     * )
@@ -243,10 +263,14 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then
       exit 2
       ;;
   esac
-  header "Doing Xcode iOS build/tests - Debug"
-  "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test
-  header "Doing Xcode iOS build/tests - Release"
-  "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test
+  if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
+    header "Doing Xcode iOS build/tests - Debug"
+    "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test
+  fi
+  if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
+    header "Doing Xcode iOS build/tests - Release"
+    "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test
+  fi
   # Don't leave the simulator in the developer's face.
   killall "${IOS_SIMULATOR_NAME}"
 fi
@@ -258,13 +282,18 @@ if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then
       # Since the ObjC 2.0 Runtime is required, 32bit OS X isn't supported.
       -destination "platform=OS X,arch=x86_64" # 64bit
   )
-  header "Doing Xcode OS X build/tests - Debug"
-  "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Debug test
-  header "Doing Xcode OS X build/tests - Release"
-  "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Release test
+  if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
+    header "Doing Xcode OS X build/tests - Debug"
+    "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Debug test
+  fi
+  if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
+    header "Doing Xcode OS X build/tests - Release"
+    "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Release test
+  fi
 fi
 
 if [[ "${DO_OBJC_CONFORMANCE_TESTS}" == "yes" ]] ; then
+  header "Running ObjC Conformance Tests"
   cd conformance
   wrapped_make -j "${NUM_MAKE_JOBS}" test_objc
   cd ..

tests.sh

@@ -197,59 +197,30 @@ internal_install_python_deps() {
   fi
 }
 
-internal_objectivec_common () {
-  # Make sure xctool is up to date. Adapted from
-  #  http://docs.travis-ci.com/user/osx-ci-environment/
-  # We don't use a before_install because we test multiple OSes.
-  brew update
-  brew outdated xctool || brew upgrade xctool
+build_objectivec_ios() {
   # Reused the build script that takes care of configuring and ensuring things
-  # are up to date. Xcode and conformance tests will be directly invoked.
+  # are up to date.  The OS X test runs the objc conformance test, so skip it
+  # here.
+  # Note: travis has xctool installed, and we've looked at using it in the past
+  # but it has ended up proving unreliable (bugs), an they are removing build
+  # support in favor of xcbuild (or just xcodebuild).
   objectivec/DevTools/full_mac_build.sh \
-      --core-only --skip-xcode --skip-objc-conformance
+      --core-only --skip-xcode-osx --skip-objc-conformance "$@"
 }
 
-internal_xctool_debug_and_release() {
-  # Always use -reporter plain to avoid escape codes in output (makes travis
-  # logs easier to read).
-  xctool -reporter plain -configuration Debug "$@"
-  xctool -reporter plain -configuration Release "$@"
+build_objectivec_ios_debug() {
+  build_objectivec_ios --skip-xcode-release
 }
 
-build_objectivec_ios() {
-  internal_objectivec_common
-  # https://github.com/facebook/xctool/issues/509 - unlike xcodebuild, xctool
-  # doesn't support >1 destination, so we have to build first and then run the
-  # tests one destination at a time.
-  internal_xctool_debug_and_release \
-    -project objectivec/ProtocolBuffers_iOS.xcodeproj \
-    -scheme ProtocolBuffers \
-    -sdk iphonesimulator \
-    build-tests
-  IOS_DESTINATIONS=(
-    "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-    "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
-    "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
-    "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
-  )
-  for i in "${IOS_DESTINATIONS[@]}" ; do
-    internal_xctool_debug_and_release \
-      -project objectivec/ProtocolBuffers_iOS.xcodeproj \
-      -scheme ProtocolBuffers \
-      -sdk iphonesimulator \
-      -destination "${i}" \
-      run-tests
-  done
+build_objectivec_ios_release() {
+  build_objectivec_ios --skip-xcode-debug
 }
 
 build_objectivec_osx() {
-  internal_objectivec_common
-  internal_xctool_debug_and_release \
-    -project objectivec/ProtocolBuffers_OSX.xcodeproj \
-    -scheme ProtocolBuffers \
-    -destination "platform=OS X,arch=x86_64" \
-    test
-  cd conformance && make test_objc && cd ..
+  # Reused the build script that takes care of configuring and ensuring things
+  # are up to date.
+  objectivec/DevTools/full_mac_build.sh \
+      --core-only --skip-xcode-ios
 }
 
 build_python() {
@@ -330,6 +301,8 @@ Usage: $0 { cpp |
             javanano_jdk7 |
             javanano_oracle7 |
             objectivec_ios |
+            objectivec_ios_debug |
+            objectivec_ios_release |
             objectivec_osx |
             python |
             python_cpp |