|  | @@ -124,6 +124,12 @@ argp.add_argument(
 | 
	
		
			
				|  |  |      help='Command to launch xDS test client. {server_uri}, {stats_port} and '
 | 
	
		
			
				|  |  |      '{qps} references will be replaced using str.format(). GRPC_XDS_BOOTSTRAP '
 | 
	
		
			
				|  |  |      'will be set for the command')
 | 
	
		
			
				|  |  | +argp.add_argument(
 | 
	
		
			
				|  |  | +    '--client_hosts',
 | 
	
		
			
				|  |  | +    default=None,
 | 
	
		
			
				|  |  | +    help='Comma-separated list of hosts running client processes. If set, '
 | 
	
		
			
				|  |  | +    '--client_cmd is ignored and client processes are assumed to be running on '
 | 
	
		
			
				|  |  | +    'the specified hosts.')
 | 
	
		
			
				|  |  |  argp.add_argument('--zone', default='us-central1-a')
 | 
	
		
			
				|  |  |  argp.add_argument('--secondary_zone',
 | 
	
		
			
				|  |  |                    default='us-west1-b',
 | 
	
	
		
			
				|  | @@ -221,6 +227,10 @@ args = argp.parse_args()
 | 
	
		
			
				|  |  |  if args.verbose:
 | 
	
		
			
				|  |  |      logger.setLevel(logging.DEBUG)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | +CLIENT_HOSTS = []
 | 
	
		
			
				|  |  | +if args.client_hosts:
 | 
	
		
			
				|  |  | +    CLIENT_HOSTS = args.client_hosts.split(',')
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  |  _DEFAULT_SERVICE_PORT = 80
 | 
	
		
			
				|  |  |  _WAIT_FOR_BACKEND_SEC = args.wait_for_backend_sec
 | 
	
		
			
				|  |  |  _WAIT_FOR_OPERATION_SEC = 1200
 | 
	
	
		
			
				|  | @@ -281,17 +291,25 @@ _SPONGE_XML_NAME = 'sponge_log.xml'
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  def get_client_stats(num_rpcs, timeout_sec):
 | 
	
		
			
				|  |  | -    with grpc.insecure_channel('localhost:%d' % args.stats_port) as channel:
 | 
	
		
			
				|  |  | -        stub = test_pb2_grpc.LoadBalancerStatsServiceStub(channel)
 | 
	
		
			
				|  |  | -        request = messages_pb2.LoadBalancerStatsRequest()
 | 
	
		
			
				|  |  | -        request.num_rpcs = num_rpcs
 | 
	
		
			
				|  |  | -        request.timeout_sec = timeout_sec
 | 
	
		
			
				|  |  | -        rpc_timeout = timeout_sec + _CONNECTION_TIMEOUT_SEC
 | 
	
		
			
				|  |  | -        response = stub.GetClientStats(request,
 | 
	
		
			
				|  |  | -                                       wait_for_ready=True,
 | 
	
		
			
				|  |  | -                                       timeout=rpc_timeout)
 | 
	
		
			
				|  |  | -        logger.debug('Invoked GetClientStats RPC: %s', response)
 | 
	
		
			
				|  |  | -        return response
 | 
	
		
			
				|  |  | +    if CLIENT_HOSTS:
 | 
	
		
			
				|  |  | +        hosts = CLIENT_HOSTS
 | 
	
		
			
				|  |  | +    else:
 | 
	
		
			
				|  |  | +        hosts = ['localhost']
 | 
	
		
			
				|  |  | +    for host in hosts:
 | 
	
		
			
				|  |  | +        with grpc.insecure_channel('%s:%d' %
 | 
	
		
			
				|  |  | +                                   (host, args.stats_port)) as channel:
 | 
	
		
			
				|  |  | +            stub = test_pb2_grpc.LoadBalancerStatsServiceStub(channel)
 | 
	
		
			
				|  |  | +            request = messages_pb2.LoadBalancerStatsRequest()
 | 
	
		
			
				|  |  | +            request.num_rpcs = num_rpcs
 | 
	
		
			
				|  |  | +            request.timeout_sec = timeout_sec
 | 
	
		
			
				|  |  | +            rpc_timeout = timeout_sec + _CONNECTION_TIMEOUT_SEC
 | 
	
		
			
				|  |  | +            logger.debug('Invoking GetClientStats RPC to %s:%d:', host,
 | 
	
		
			
				|  |  | +                         args.stats_port)
 | 
	
		
			
				|  |  | +            response = stub.GetClientStats(request,
 | 
	
		
			
				|  |  | +                                           wait_for_ready=True,
 | 
	
		
			
				|  |  | +                                           timeout=rpc_timeout)
 | 
	
		
			
				|  |  | +            logger.debug('Invoked GetClientStats RPC to %s: %s', host, response)
 | 
	
		
			
				|  |  | +            return response
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  class RpcDistributionError(Exception):
 | 
	
	
		
			
				|  | @@ -1649,7 +1667,11 @@ try:
 | 
	
		
			
				|  |  |      gcp_suffix = args.gcp_suffix
 | 
	
		
			
				|  |  |      health_check_name = _BASE_HEALTH_CHECK_NAME + gcp_suffix
 | 
	
		
			
				|  |  |      if not args.use_existing_gcp_resources:
 | 
	
		
			
				|  |  | -        num_attempts = 5
 | 
	
		
			
				|  |  | +        if gcp_suffix:
 | 
	
		
			
				|  |  | +            num_attempts = 5
 | 
	
		
			
				|  |  | +        else:
 | 
	
		
			
				|  |  | +            # If not given a suffix, do not retry if already in use.
 | 
	
		
			
				|  |  | +            num_attempts = 1
 | 
	
		
			
				|  |  |          for i in range(num_attempts):
 | 
	
		
			
				|  |  |              try:
 | 
	
		
			
				|  |  |                  logger.info('Using GCP suffix %s', gcp_suffix)
 | 
	
	
		
			
				|  | @@ -1792,20 +1814,21 @@ try:
 | 
	
		
			
				|  |  |              # resources).
 | 
	
		
			
				|  |  |              fail_on_failed_rpc = ''
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -            client_cmd_formatted = args.client_cmd.format(
 | 
	
		
			
				|  |  | -                server_uri=server_uri,
 | 
	
		
			
				|  |  | -                stats_port=args.stats_port,
 | 
	
		
			
				|  |  | -                qps=args.qps,
 | 
	
		
			
				|  |  | -                fail_on_failed_rpc=fail_on_failed_rpc,
 | 
	
		
			
				|  |  | -                rpcs_to_send=rpcs_to_send,
 | 
	
		
			
				|  |  | -                metadata_to_send=metadata_to_send)
 | 
	
		
			
				|  |  | -            logger.debug('running client: %s', client_cmd_formatted)
 | 
	
		
			
				|  |  | -            client_cmd = shlex.split(client_cmd_formatted)
 | 
	
		
			
				|  |  |              try:
 | 
	
		
			
				|  |  | -                client_process = subprocess.Popen(client_cmd,
 | 
	
		
			
				|  |  | -                                                  env=client_env,
 | 
	
		
			
				|  |  | -                                                  stderr=subprocess.STDOUT,
 | 
	
		
			
				|  |  | -                                                  stdout=test_log_file)
 | 
	
		
			
				|  |  | +                if not CLIENT_HOSTS:
 | 
	
		
			
				|  |  | +                    client_cmd_formatted = args.client_cmd.format(
 | 
	
		
			
				|  |  | +                        server_uri=server_uri,
 | 
	
		
			
				|  |  | +                        stats_port=args.stats_port,
 | 
	
		
			
				|  |  | +                        qps=args.qps,
 | 
	
		
			
				|  |  | +                        fail_on_failed_rpc=fail_on_failed_rpc,
 | 
	
		
			
				|  |  | +                        rpcs_to_send=rpcs_to_send,
 | 
	
		
			
				|  |  | +                        metadata_to_send=metadata_to_send)
 | 
	
		
			
				|  |  | +                    logger.debug('running client: %s', client_cmd_formatted)
 | 
	
		
			
				|  |  | +                    client_cmd = shlex.split(client_cmd_formatted)
 | 
	
		
			
				|  |  | +                    client_process = subprocess.Popen(client_cmd,
 | 
	
		
			
				|  |  | +                                                      env=client_env,
 | 
	
		
			
				|  |  | +                                                      stderr=subprocess.STDOUT,
 | 
	
		
			
				|  |  | +                                                      stdout=test_log_file)
 | 
	
		
			
				|  |  |                  if test_case == 'backends_restart':
 | 
	
		
			
				|  |  |                      test_backends_restart(gcp, backend_service, instance_group)
 | 
	
		
			
				|  |  |                  elif test_case == 'change_backend_service':
 | 
	
	
		
			
				|  | @@ -1847,7 +1870,7 @@ try:
 | 
	
		
			
				|  |  |                  else:
 | 
	
		
			
				|  |  |                      logger.error('Unknown test case: %s', test_case)
 | 
	
		
			
				|  |  |                      sys.exit(1)
 | 
	
		
			
				|  |  | -                if client_process.poll() is not None:
 | 
	
		
			
				|  |  | +                if client_process and client_process.poll() is not None:
 | 
	
		
			
				|  |  |                      raise Exception(
 | 
	
		
			
				|  |  |                          'Client process exited prematurely with exit code %d' %
 | 
	
		
			
				|  |  |                          client_process.returncode)
 | 
	
	
		
			
				|  | @@ -1859,8 +1882,12 @@ try:
 | 
	
		
			
				|  |  |                  result.state = 'FAILED'
 | 
	
		
			
				|  |  |                  result.message = str(e)
 | 
	
		
			
				|  |  |              finally:
 | 
	
		
			
				|  |  | -                if client_process and not client_process.returncode:
 | 
	
		
			
				|  |  | -                    client_process.terminate()
 | 
	
		
			
				|  |  | +                if client_process:
 | 
	
		
			
				|  |  | +                    if client_process.returncode:
 | 
	
		
			
				|  |  | +                        logger.info('Client exited with code %d' %
 | 
	
		
			
				|  |  | +                                    client_process.returncode)
 | 
	
		
			
				|  |  | +                    else:
 | 
	
		
			
				|  |  | +                        client_process.terminate()
 | 
	
		
			
				|  |  |                  test_log_file.close()
 | 
	
		
			
				|  |  |                  # Workaround for Python 3, as report_utils will invoke decode() on
 | 
	
		
			
				|  |  |                  # result.message, which has a default value of ''.
 |