| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304 | #!/usr/bin/env python2.7# Copyright 2017 gRPC authors.## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at##     http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License."""Build and upload docker images to Google Container Registry per matrix."""from __future__ import print_functionimport argparseimport atexitimport multiprocessingimport osimport shutilimport subprocessimport sysimport tempfile# Langauage Runtime Matriximport client_matrixpython_util_dir = os.path.abspath(os.path.join(    os.path.dirname(__file__), '../run_tests/python_utils'))sys.path.append(python_util_dir)import dockerjobimport jobset_IMAGE_BUILDER = 'tools/run_tests/dockerize/build_interop_image.sh'_LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()# All gRPC release tags, flattened, deduped and sorted._RELEASES = sorted(list(set(    client_matrix.get_release_tag_name(info) for lang in client_matrix.LANG_RELEASE_MATRIX.values() for info in lang)))# Destination directory inside docker image to keep extra info from build time._BUILD_INFO = '/var/local/build_info'argp = argparse.ArgumentParser(description='Run interop tests.')argp.add_argument('--gcr_path',                  default='gcr.io/grpc-testing',                  help='Path of docker images in Google Container Registry')argp.add_argument('--release',                  default='master',                  choices=['all', 'master'] + _RELEASES,                  help='github commit tag to checkout.  When building all '                  'releases defined in client_matrix.py, use "all". Valid only '                  'with --git_checkout.')argp.add_argument('-l', '--language',                  choices=['all'] + sorted(_LANGUAGES),                  nargs='+',                  default=['all'],                  help='Test languages to build docker images for.')argp.add_argument('--git_checkout',                  action='store_true',                  help='Use a separate git clone tree for building grpc stack. '                  'Required when using --release flag.  By default, current'                  'tree and the sibling will be used for building grpc stack.')argp.add_argument('--git_checkout_root',                  default='/export/hda3/tmp/grpc_matrix',                  help='Directory under which grpc-go/java/main repo will be '                  'cloned.  Valid only with --git_checkout.')argp.add_argument('--keep',                  action='store_true',                  help='keep the created local images after uploading to GCR')argp.add_argument('--reuse_git_root',                  default=False,                  action='store_const',                  const=True,                                    help='reuse the repo dir. If False, the existing git root '                  'directory will removed before a clean checkout, because '                  'reusing the repo can cause git checkout error if you switch '                  'between releases.')args = argp.parse_args()def add_files_to_image(image, with_files, label=None):  """Add files to a docker image.  image: docker image name, i.e. grpc_interop_java:26328ad8  with_files: additional files to include in the docker image.  label: label string to attach to the image.  """  tag_idx = image.find(':')  if tag_idx == -1:    jobset.message('FAILED', 'invalid docker image %s' % image, do_newline=True)    sys.exit(1)  orig_tag = '%s_' % image  subprocess.check_output(['docker', 'tag', image, orig_tag])  lines = ['FROM ' + orig_tag]  if label:    lines.append('LABEL %s' % label)  temp_dir = tempfile.mkdtemp()  atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir]))  # Copy with_files inside the tmp directory, which will be the docker build  # context.  for f in with_files:    shutil.copy(f, temp_dir)    lines.append('COPY %s %s/' % (os.path.basename(f), _BUILD_INFO))  # Create a Dockerfile.  with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f:    f.write('\n'.join(lines))  jobset.message('START', 'Repackaging %s' % image, do_newline=True)  build_cmd = ['docker', 'build', '--rm', '--tag', image, temp_dir]  subprocess.check_output(build_cmd)  dockerjob.remove_image(orig_tag, skip_nonexistent=True)def build_image_jobspec(runtime, env, gcr_tag, stack_base):  """Build interop docker image for a language with runtime.  runtime: a <lang><version> string, for example go1.8.  env:     dictionary of env to passed to the build script.  gcr_tag: the tag for the docker image (i.e. v1.3.0).  stack_base: the local gRPC repo path.  """  basename = 'grpc_interop_%s' % runtime  tag = '%s/%s:%s' % (args.gcr_path, basename, gcr_tag)  build_env = {      'INTEROP_IMAGE': tag,      'BASE_NAME': basename,      'TTY_FLAG': '-t'  }  build_env.update(env)  image_builder_path = _IMAGE_BUILDER  if client_matrix.should_build_docker_interop_image_from_release_tag(lang):    image_builder_path = os.path.join(stack_base, _IMAGE_BUILDER)  build_job = jobset.JobSpec(          cmdline=[image_builder_path],          environ=build_env,          shortname='build_docker_%s' % runtime,          timeout_seconds=30*60)  build_job.tag = tag  return build_jobdef build_all_images_for_lang(lang):  """Build all docker images for a language across releases and runtimes."""  if not args.git_checkout:    if args.release != 'master':      print('WARNING: --release is set but will be ignored\n')    releases = ['master']  else:    if args.release == 'all':      releases = client_matrix.get_release_tags(lang)    else:      # Build a particular release.      if args.release not in ['master'] + client_matrix.get_release_tags(lang):        jobset.message('SKIPPED',                       '%s for %s is not defined' % (args.release, lang),                       do_newline=True)        return []      releases = [args.release]  images = []  for release in releases:    images += build_all_images_for_release(lang, release)  jobset.message('SUCCESS',                 'All docker images built for %s at %s.' % (lang, releases),                 do_newline=True)  return imagesdef build_all_images_for_release(lang, release):  """Build all docker images for a release across all runtimes."""  docker_images = []  build_jobs = []  env = {}  # If we not using current tree or the sibling for grpc stack, do checkout.  stack_base = ''  if args.git_checkout:    stack_base = checkout_grpc_stack(lang, release)    var ={'go': 'GRPC_GO_ROOT', 'java': 'GRPC_JAVA_ROOT', 'node': 'GRPC_NODE_ROOT'}.get(lang, 'GRPC_ROOT')    env[var] = stack_base  for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:    job = build_image_jobspec(runtime, env, release, stack_base)    docker_images.append(job.tag)    build_jobs.append(job)  jobset.message('START', 'Building interop docker images.', do_newline=True)  print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))  num_failures, _ = jobset.run(      build_jobs, newline_on_success=True, maxjobs=multiprocessing.cpu_count())  if num_failures:    jobset.message('FAILED', 'Failed to build interop docker images.',                   do_newline=True)    docker_images_cleanup.extend(docker_images)    sys.exit(1)  jobset.message('SUCCESS',                 'All docker images built for %s at %s.' % (lang, release),                 do_newline=True)  if release != 'master':    commit_log = os.path.join(stack_base, 'commit_log')    if os.path.exists(commit_log):      for image in docker_images:        add_files_to_image(image, [commit_log], 'release=%s' % release)  return docker_imagesdef cleanup():  if not args.keep:    for image in docker_images_cleanup:      dockerjob.remove_image(image, skip_nonexistent=True)docker_images_cleanup = []atexit.register(cleanup)def maybe_apply_patches_on_git_tag(stack_base, lang, release):  files_to_patch = []  for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]:    if client_matrix.get_release_tag_name(release_info) == release:      files_to_patch = release_info[release].get('patch')      break  if not files_to_patch:    return  patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release)  patch_file = os.path.abspath(os.path.join(os.path.dirname(__file__),                                            patch_file_relative_path))  if not os.path.exists(patch_file):    jobset.message('FAILED', 'expected patch file |%s| to exist' % patch_file)    sys.exit(1)  subprocess.check_output(      ['git', 'apply', patch_file], cwd=stack_base, stderr=subprocess.STDOUT)  for repo_relative_path in files_to_patch:    subprocess.check_output(        ['git', 'add', repo_relative_path],        cwd=stack_base,        stderr=subprocess.STDOUT)  subprocess.check_output(      ['git', 'commit', '-m', ('Hack performed on top of %s git '                               'tag in order to build and run the %s '                               'interop tests on that tag.' % (lang, release))],      cwd=stack_base, stderr=subprocess.STDOUT)def checkout_grpc_stack(lang, release):  """Invokes 'git check' for the lang/release and returns directory created."""  assert args.git_checkout and args.git_checkout_root  if not os.path.exists(args.git_checkout_root):    os.makedirs(args.git_checkout_root)  repo = client_matrix.get_github_repo(lang)  # Get the subdir name part of repo  # For example, 'git@github.com:grpc/grpc-go.git' should use 'grpc-go'.  repo_dir = os.path.splitext(os.path.basename(repo))[0]  stack_base = os.path.join(args.git_checkout_root, repo_dir)  # Clean up leftover repo dir if necessary.  if not args.reuse_git_root and os.path.exists(stack_base):    jobset.message('START', 'Removing git checkout root.', do_newline=True)    shutil.rmtree(stack_base)  if not os.path.exists(stack_base):    subprocess.check_call(['git', 'clone', '--recursive', repo],                          cwd=os.path.dirname(stack_base))  # git checkout.  jobset.message('START', 'git checkout %s from %s' % (release, stack_base),                 do_newline=True)  # We should NEVER do checkout on current tree !!!  assert not os.path.dirname(__file__).startswith(stack_base)  output = subprocess.check_output(      ['git', 'checkout', release], cwd=stack_base, stderr=subprocess.STDOUT)  maybe_apply_patches_on_git_tag(stack_base, lang, release)  commit_log = subprocess.check_output(['git', 'log', '-1'], cwd=stack_base)  jobset.message('SUCCESS', 'git checkout',                  '%s: %s' % (str(output), commit_log),                  do_newline=True)  # Write git log to commit_log so it can be packaged with the docker image.  with open(os.path.join(stack_base, 'commit_log'), 'w') as f:    f.write(commit_log)  return stack_baselanguages = args.language if args.language != ['all'] else _LANGUAGESfor lang in languages:  docker_images = build_all_images_for_lang(lang)  for image in docker_images:    jobset.message('START', 'Uploading %s' % image, do_newline=True)    # docker image name must be in the format <gcr_path>/<image>:<gcr_tag>    assert image.startswith(args.gcr_path) and image.find(':') != -1    subprocess.call(['gcloud', 'docker', '--', 'push', image])
 |