Backport #25647 and #25510 (#25664)

* Fix pypa link (#25647)

* Fix pypa link

* And 3.5 as well

* [xDS Proto] Pin PyPI dependencies in Dockerfiles (#25510)

* Regenerate projects

Co-authored-by: Lidi Zheng <lidiz@google.com>

templates/tools/dockerfile/apt_get_python_27.include

@@ -1,3 +1,3 @@
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7

templates/tools/dockerfile/gcp_api_libraries.include

@@ -1,2 +1,2 @@
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0

templates/tools/dockerfile/python_deps.include

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y ${'\\'}
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

templates/tools/dockerfile/test/python_stretch_3.5_x64/Dockerfile.template

@@ -17,4 +17,4 @@
   <%include file="../../python_stretch.include"/>
 
   RUN apt-get update && apt-get install -y python3.5 python3-all-dev
-  RUN curl https://bootstrap.pypa.io/3.5/get-pip.py | python3.5
+  RUN curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | python3.5

templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template

@@ -19,7 +19,7 @@
   <%include file="../../compile_python_38.include"/>
 
   RUN apt-get update && apt-get install -y python3.5 python3.5-dev
-  RUN curl https://bootstrap.pypa.io/3.5/get-pip.py | python3.5
+  RUN curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | python3.5
 
   RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev
   RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7

templates/tools/dockerfile/test/sanity/Dockerfile.template

@@ -33,6 +33,9 @@
   RUN python2 -m pip install simplejson mako virtualenv==16.7.9 lxml
   RUN python3 -m pip install simplejson mako virtualenv==16.7.9 lxml six
 
+  # Upgrade Python's YAML library
+  RUN python3 -m pip install --upgrade --ignore-installed PyYAML==5.4.1 --user
+
   # Add buster-backports for more recent clang packages
   RUN echo "deb http://deb.debian.org/debian buster-backports main" | tee /etc/apt/sources.list.d/buster-backports.list
 

tools/buildgen/generate_projects.sh

@@ -18,9 +18,6 @@ set -e
 
 export TEST=${TEST:-false}
 
-# Upgrade Python's YAML library
-python3 -m pip install --upgrade --ignore-installed PyYAML --user
-
 echo "Generating build_autogenerated.yaml from bazel BUILD file"
 rm -f build_autogenerated.yaml
 python3 tools/buildgen/extract_metadata_from_bazel_xml.py

tools/dockerfile/grpc_clang_tidy/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 
 RUN mkdir /var/local/jenkins

tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile

@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile

@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile

@@ -61,7 +61,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/interoptest/grpc_interop_go/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/interoptest/grpc_interop_go1.11/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/interoptest/grpc_interop_node/Dockerfile

@@ -61,7 +61,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/interoptest/grpc_interop_python/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 # Add Debian 'buster' repository, we will need it for installing newer versions of python
 RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list

tools/dockerfile/interoptest/grpc_interop_pythonasyncio/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 # Add Debian 'buster' repository, we will need it for installing newer versions of python
 RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list

tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile

@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/test/bazel/Dockerfile

@@ -50,7 +50,7 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0

tools/dockerfile/test/csharp_stretch_x64/Dockerfile

@@ -60,13 +60,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #================
 # C# dependencies

tools/dockerfile/test/cxx_buster_x64/Dockerfile

@@ -60,13 +60,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #=================
 # C++ dependencies

tools/dockerfile/test/cxx_jessie_x64/Dockerfile

@@ -61,13 +61,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #=================
 # C++ dependencies

tools/dockerfile/test/cxx_jessie_x86/Dockerfile

@@ -61,13 +61,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #=================
 # C++ dependencies

tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile

@@ -60,13 +60,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #=================
 # C++ dependencies

tools/dockerfile/test/cxx_ubuntu1804_x64/Dockerfile

@@ -60,13 +60,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #=================
 # C++ dependencies

tools/dockerfile/test/node_jessie_x64/Dockerfile

@@ -72,13 +72,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #==================
 # Node dependencies

tools/dockerfile/test/php7_jessie_x64/Dockerfile

@@ -72,13 +72,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #=================	
 # PHP Test dependencies	

tools/dockerfile/test/python_stretch_2.7_x64/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 # Add Debian 'buster' repository, we will need it for installing newer versions of python
 RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list

tools/dockerfile/test/python_stretch_3.5_x64/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 # Add Debian 'buster' repository, we will need it for installing newer versions of python
 RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
@@ -65,4 +65,4 @@ RUN mkdir /var/local/jenkins
 
 
 RUN apt-get update && apt-get install -y python3.5 python3-all-dev
-RUN curl https://bootstrap.pypa.io/3.5/get-pip.py | python3.5
+RUN curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | python3.5

tools/dockerfile/test/python_stretch_3.6_x64/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 # Add Debian 'buster' repository, we will need it for installing newer versions of python
 RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list

tools/dockerfile/test/python_stretch_3.7_x64/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 # Add Debian 'buster' repository, we will need it for installing newer versions of python
 RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list

tools/dockerfile/test/python_stretch_3.8_x64/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 # Add Debian 'buster' repository, we will need it for installing newer versions of python
 RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list

tools/dockerfile/test/python_stretch_default_x64/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 # Add Debian 'buster' repository, we will need it for installing newer versions of python
 RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
@@ -105,7 +105,7 @@ RUN python3.8 -m ensurepip && \
 
 
 RUN apt-get update && apt-get install -y python3.5 python3.5-dev
-RUN curl https://bootstrap.pypa.io/3.5/get-pip.py | python3.5
+RUN curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | python3.5
 
 RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7

tools/dockerfile/test/ruby_buster_x64/Dockerfile

@@ -60,13 +60,13 @@ RUN apt-get update && apt-get install -y \
     python-setuptools
 
 # Install Python packages from PyPI
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 RUN pip install --upgrade pip==19.3.1
 RUN pip install virtualenv==16.7.9
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.15.0 twisted==17.5.0
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 #==================
 # Ruby dependencies

tools/dockerfile/test/sanity/Dockerfile

@@ -51,10 +51,10 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Install Python 2.7
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
-RUN curl https://bootstrap.pypa.io/2.7/get-pip.py | python2.7
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2.7
 
 # Google Cloud platform API libraries
-RUN pip install --upgrade google-api-python-client oauth2client
+RUN pip install --upgrade google-auth==1.24.0 google-api-python-client==1.12.8 oauth2client==4.1.0
 
 
 RUN mkdir /var/local/jenkins
@@ -81,6 +81,9 @@ RUN apt-get update && apt-get install -y \
 RUN python2 -m pip install simplejson mako virtualenv==16.7.9 lxml
 RUN python3 -m pip install simplejson mako virtualenv==16.7.9 lxml six
 
+# Upgrade Python's YAML library
+RUN python3 -m pip install --upgrade --ignore-installed PyYAML==5.4.1 --user
+
 # Add buster-backports for more recent clang packages
 RUN echo "deb http://deb.debian.org/debian buster-backports main" | tee /etc/apt/sources.list.d/buster-backports.list
 

tools/run_tests/helper_scripts/build_python.sh

@@ -153,7 +153,7 @@ fi
 # See https://github.com/grpc/grpc/issues/14815 for more context. We cannot rely
 # on pip to upgrade itself because if pip is too old, it may not have the required
 # TLS version to run `pip install`.
-curl https://bootstrap.pypa.io/2.7/get-pip.py | $VENV_PYTHON
+curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | $VENV_PYTHON
 
 # pip-installs the directory specified. Used because on MSYS the vanilla Windows
 # Python gets confused when parsing paths.