Retry test when xcodebuild exit with simulator error

src/objective-c/tests/run_tests.sh

@@ -34,36 +34,49 @@ $BINDIR/interop_server --port=5051 --max_send_message_size=8388608 --use_tls &
 # Kill them when this script exits.
 trap 'kill -9 `jobs -p` ; echo "EXIT TIME:  $(date)"' EXIT
 
-# Boot Xcode first with several retries since Xcode might fail due to a bug:
-# http://www.openradar.me/29785686
-xcrun simctl list | egrep 'iPhone 6 \('
-udid=`xcrun simctl list | egrep 'iPhone 6 \(.*\) \(.*\)' | sed -E 's/ *iPhone 6 \(([^\)]*)\).*/\1/g' | head -n 1`
-retries=0
-while [ $retries -lt 3 ] && ! open -a Simulator --args -CurrentDeviceUDID $udid ; do
-retries=$(($retries+1))
-done
-if [ $retries == 3 ]; then
-  echo "Xcode simulator failed to start after 3 retries."
-  exit 1
-fi
+set -o pipefail
 
 # xcodebuild is very verbose. We filter its output and tell Bash to fail if any
 # element of the pipe fails.
 # TODO(jcanizales): Use xctool instead? Issue #2540.
-set -o pipefail
 XCODEBUILD_FILTER='(^CompileC |^Ld |^ *[^ ]*clang |^ *cd |^ *export |^Libtool |^ *[^ ]*libtool |^CpHeader |^ *builtin-copy )'
+
 echo "TIME:  $(date)"
-xcodebuild \
-    -workspace Tests.xcworkspace \
-    -scheme AllTests \
-    -destination name="iPhone 6" \
-    HOST_PORT_LOCALSSL=localhost:5051 \
-    HOST_PORT_LOCAL=localhost:5050 \
-    HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \
-    test \
-    | egrep -v "$XCODEBUILD_FILTER" \
-    | egrep -v '^$' \
-    | egrep -v "(GPBDictionary|GPBArray)" -
+
+# Retry the test for up to 3 times when return code is 65, due to Xcode issue:
+# http://www.openradar.me/29785686
+# The issue seems to be a connectivity issue to Xcode simulator so only retry
+# the first xcodebuild command
+retries=0
+while [ $retries -lt 3 ]; do
+  return_code=0
+  out=$(xcodebuild \
+        -workspace Tests.xcworkspace \
+        -scheme AllTests \
+        -destination name="iPhone 6" \
+        HOST_PORT_LOCALSSL=localhost:5051 \
+        HOST_PORT_LOCAL=localhost:5050 \
+        HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \
+        test \
+        | egrep -v "$XCODEBUILD_FILTER" \
+        | egrep -v '^$' \
+        | egrep -v "(GPBDictionary|GPBArray)" - ) || return_code=$?
+  if [ $return_code == 65 ] && [[ $out == *"DTXProxyChannel error 1"* ]]; then
+    echo "$out"
+    echo "Failed with code 65 (DTXProxyChannel error 1); retry."
+    retries=$(($retries+1))
+  elif [ $return_code == 0 ]; then
+    break
+  else
+    echo "$out"
+    echo "Failed with code $return_code."
+    exit 1
+  fi
+done
+if [ $retries == 3 ]; then
+  echo "Failed with code 65 for 3 times; abort."
+  exit 1
+fi
 
 echo "TIME:  $(date)"
 xcodebuild \
@@ -95,3 +108,5 @@ xcodebuild \
     | egrep -v "$XCODEBUILD_FILTER" \
     | egrep -v '^$' \
     | egrep -v "(GPBDictionary|GPBArray)" -
+
+exit 0