Merge pull request #25079 from grpc/xds-k8s-server-authz-error-test

xds-k8s driver: implement PSM security server-authz-error test

tools/run_tests/xds_k8s_test_driver/bin/run_channelz.py

@@ -18,10 +18,11 @@ as a part of interop test suites.
 
 Typical usage examples:
 
-    # Show channel and socket info
+    # Show channel and server socket pair
     python -m bin.run_channelz --flagfile=config/local-dev.cfg
 
-    # Evaluate setup for mtls_error test case
+    # Evaluate setup for different security configurations
+    python -m bin.run_channelz --flagfile=config/local-dev.cfg --security=tls
     python -m bin.run_channelz --flagfile=config/local-dev.cfg --security=mtls_error
 
     # More information and usage options
@@ -49,9 +50,12 @@ _CLIENT_RPC_HOST = flags.DEFINE_string('client_rpc_host',
                                        default='127.0.0.1',
                                        help='Client RPC host')
 _SECURITY = flags.DEFINE_enum('security',
-                              default='positive_cases',
-                              enum_values=['positive_cases', 'mtls_error'],
-                              help='Test for security setup')
+                              default=None,
+                              enum_values=[
+                                  'mtls', 'tls', 'plaintext', 'mtls_error',
+                                  'server_authz_error'
+                              ],
+                              help='Show info for a security setup')
 flags.adopt_module_key_flags(xds_flags)
 flags.adopt_module_key_flags(xds_k8s_flags)
 
@@ -81,10 +85,13 @@ def get_deployment_pod_ips(k8s_ns, deployment_name):
     return [pod.status.pod_ip for pod in pods]
 
 
-def negative_case_mtls(test_client, test_server):
-    """Debug mTLS Error case.
+def debug_security_setup_negative(test_client):
+    """Debug negative cases: mTLS Error, Server AuthZ error
 
-    Server expects client mTLS cert, but client configured only for TLS.
+    1) mTLS Error: Server expects client mTLS cert,
+       but client configured only for TLS.
+    2) AuthZ error: Client does not authorize server because of mismatched
+       SAN name.
     """
     # Client side.
     client_correct_setup = True
@@ -94,7 +101,7 @@ def negative_case_mtls(test_client, test_server):
         subchannel, *subchannels = list(
             test_client.channelz.list_channel_subchannels(channel))
     except ValueError:
-        print("(mTLS-error) Client setup fail: subchannel not found. "
+        print("Client setup fail: subchannel not found. "
               "Common causes: test client didn't connect to TD; "
               "test client exhausted retries, and closed all subchannels.")
         return
@@ -103,27 +110,27 @@ def negative_case_mtls(test_client, test_server):
     logger.debug('Found subchannel, %s', subchannel)
     if subchannels:
         client_correct_setup = False
-        print(f'(mTLS-error) Unexpected subchannels {subchannels}')
+        print(f'Unexpected subchannels {subchannels}')
     subchannel_state: _ChannelState = subchannel.data.state.state
     if subchannel_state is not _ChannelState.TRANSIENT_FAILURE:
         client_correct_setup = False
-        print('(mTLS-error) Subchannel expected to be in '
+        print('Subchannel expected to be in '
               'TRANSIENT_FAILURE, same as its channel')
 
     # Client subchannel must have no sockets.
     sockets = list(test_client.channelz.list_subchannels_sockets(subchannel))
     if sockets:
         client_correct_setup = False
-        print(f'(mTLS-error) Unexpected subchannel sockets {sockets}')
+        print(f'Unexpected subchannel sockets {sockets}')
 
     # Results.
     if client_correct_setup:
-        print('(mTLS-error) Client setup pass: the channel '
+        print('Client setup pass: the channel '
               'to the server has exactly one subchannel '
               'in TRANSIENT_FAILURE, and no sockets')
 
 
-def positive_case_all(test_client, test_server):
+def debug_security_setup_positive(test_client, test_server):
     """Debug positive cases: mTLS, TLS, Plaintext."""
     test_client.wait_for_active_server_channel()
     client_sock: _Socket = test_client.get_active_server_channel_socket()
@@ -150,6 +157,17 @@ def positive_case_all(test_client, test_server):
         print('(mTLS) Not detected')
 
 
+def debug_basic_setup(test_client, test_server):
+    """Show channel and server socket pair"""
+    test_client.wait_for_active_server_channel()
+    client_sock: _Socket = test_client.get_active_server_channel_socket()
+    server_sock: _Socket = test_server.get_server_socket_matching_client(
+        client_sock)
+
+    print(f'Client socket:\n{client_sock}\n')
+    print(f'Matching server:\n{server_sock}\n')
+
+
 def main(argv):
     if len(argv) > 1:
         raise app.UsageError('Too many command-line arguments.')
@@ -179,11 +197,12 @@ def main(argv):
         rpc_port=xds_flags.CLIENT_PORT.value,
         rpc_host=_CLIENT_RPC_HOST.value)
 
-    # Run checks
-    if _SECURITY.value in 'positive_cases':
-        positive_case_all(test_client, test_server)
-    elif _SECURITY.value == 'mtls_error':
-        negative_case_mtls(test_client, test_server)
+    if _SECURITY.value in ('mtls', 'tls', 'plaintext'):
+        debug_security_setup_positive(test_client, test_server)
+    elif _SECURITY.value == ('mtls_error', 'server_authz_error'):
+        debug_security_setup_negative(test_client)
+    else:
+        debug_basic_setup(test_client, test_server)
 
     test_client.close()
     test_server.close()

tools/run_tests/xds_k8s_test_driver/bin/run_td_setup.py

@@ -31,6 +31,7 @@ Typical usage examples:
     python -m bin.run_td_setup --helpfull
 """
 import logging
+import uuid
 
 from absl import app
 from absl import flags
@@ -50,11 +51,13 @@ _CMD = flags.DEFINE_enum('cmd',
                              'backends-cleanup'
                          ],
                          help='Command')
-_SECURITY = flags.DEFINE_enum(
-    'security',
-    default=None,
-    enum_values=['mtls', 'tls', 'plaintext', 'mtls_error'],
-    help='Configure TD with security')
+_SECURITY = flags.DEFINE_enum('security',
+                              default=None,
+                              enum_values=[
+                                  'mtls', 'tls', 'plaintext', 'mtls_error',
+                                  'server_authz_error'
+                              ],
+                              help='Configure TD with security')
 flags.adopt_module_key_flags(xds_flags)
 flags.adopt_module_key_flags(xds_k8s_flags)
 
@@ -151,6 +154,24 @@ def main(argv):
                                          tls=True,
                                          mtls=False)
 
+            elif security_mode == 'server_authz_error':
+                # Error case: client does not authorize server
+                # because of mismatched SAN name.
+                logger.info('Setting up mtls_error')
+                td.setup_for_grpc(server_xds_host, server_xds_port)
+                # Regular TLS setup, but with client policy configured using
+                # intentionality incorrect server_namespace.
+                td.setup_server_security(server_namespace=namespace,
+                                         server_name=server_name,
+                                         server_port=server_port,
+                                         tls=True,
+                                         mtls=False)
+                incorrect_namespace = f'incorrect-namespace-{uuid.uuid4().hex}'
+                td.setup_client_security(server_namespace=incorrect_namespace,
+                                         server_name=server_name,
+                                         tls=True,
+                                         mtls=False)
+
             logger.info('Works!')
     except Exception:  # noqa pylint: disable=broad-except
         logger.exception('Got error during creation')

tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py

@@ -11,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import datetime
 import enum
 import hashlib
 import logging
+import time
 from typing import Optional, Tuple
 
 from absl import flags
@@ -42,6 +44,7 @@ XdsTestServer = server_app.XdsTestServer
 XdsTestClient = client_app.XdsTestClient
 LoadBalancerStatsResponse = grpc_testing.LoadBalancerStatsResponse
 _ChannelState = grpc_channelz.ChannelState
+_timedelta = datetime.timedelta
 
 
 class XdsKubernetesTestCase(absltest.TestCase):
@@ -123,7 +126,11 @@ class XdsKubernetesTestCase(absltest.TestCase):
                              num_rpcs: int = 100):
         lb_stats = self.sendRpcs(test_client, num_rpcs)
         self.assertAllBackendsReceivedRpcs(lb_stats)
-        self.assertFailedRpcsAtMost(lb_stats, 0)
+        failed = int(lb_stats.num_failures)
+        self.assertLessEqual(
+            failed,
+            0,
+            msg=f'Expected all RPCs to succeed: {failed} of {num_rpcs} failed')
 
     def assertFailedRpcs(self,
                          test_client: XdsTestClient,
@@ -133,7 +140,7 @@ class XdsKubernetesTestCase(absltest.TestCase):
         self.assertEqual(
             failed,
             num_rpcs,
-            msg=f'Expected all {num_rpcs} RPCs to fail, but {failed} failed')
+            msg=f'Expected all RPCs to fail: {failed} of {num_rpcs} failed')
 
     @staticmethod
     def sendRpcs(test_client: XdsTestClient,
@@ -152,13 +159,6 @@ class XdsKubernetesTestCase(absltest.TestCase):
                 0,
                 msg=f'Backend {backend} did not receive a single RPC')
 
-    def assertFailedRpcsAtMost(self, lb_stats, limit):
-        failed = int(lb_stats.num_failures)
-        self.assertLessEqual(
-            failed,
-            limit,
-            msg=f'Unexpected number of RPC failures {failed} > {limit}')
-
 
 class RegularXdsKubernetesTestCase(XdsKubernetesTestCase):
 
@@ -337,9 +337,6 @@ class SecurityXdsKubernetesTestCase(XdsKubernetesTestCase):
             msg="(mTLS) Server remote certificate must match client's "
             "local certificate")
 
-        # Success
-        logger.info('mTLS security mode  confirmed!')
-
     def assertSecurityTls(self, client_security: grpc_channelz.Security,
                           server_security: grpc_channelz.Security):
         self.assertEqual(client_security.WhichOneof('model'),
@@ -370,9 +367,6 @@ class SecurityXdsKubernetesTestCase(XdsKubernetesTestCase):
             msg="(TLS) Client local certificate must be empty in TLS mode. "
             "Is client security incorrectly configured for mTLS?")
 
-        # Success
-        logger.info('TLS security mode confirmed!')
-
     def assertSecurityPlaintext(self, client_security, server_security):
         server_tls, client_tls = server_security.tls, client_security.tls
         # Not TLS
@@ -391,10 +385,46 @@ class SecurityXdsKubernetesTestCase(XdsKubernetesTestCase):
             client_tls.local_certificate,
             msg="(Plaintext) Client local certificate must be empty.")
 
-        # Success
-        logger.info('Plaintext security mode confirmed!')
-
-    def assertMtlsErrorSetup(self, test_client: XdsTestClient):
+    def assertClientCannotReachServerRepeatedly(
+            self,
+            test_client: XdsTestClient,
+            *,
+            times: Optional[int] = None,
+            delay: Optional[_timedelta] = None):
+        """
+        Asserts that the client repeatedly cannot reach the server.
+
+        With negative tests we can't be absolutely certain expected failure
+        state is not caused by something else.
+        To mitigate for this, we repeat the checks several times, and expect
+        all of them to succeed.
+
+        This is useful in case the channel eventually stabilizes, and RPCs pass.
+
+        Args:
+            test_client: An instance of XdsTestClient
+            times: Optional; A positive number of times to confirm that
+                the server is unreachable. Defaults to `3` attempts.
+            delay: Optional; Specifies how long to wait before the next check.
+                Defaults to `10` seconds.
+        """
+        if times is None or times < 1:
+            times = 3
+        if delay is None:
+            delay = _timedelta(seconds=10)
+
+        for i in range(1, times + 1):
+            self.assertClientCannotReachServer(test_client)
+            if i < times:
+                logger.info('Check %s passed, waiting %s before the next check',
+                            i, delay)
+                time.sleep(delay.total_seconds())
+
+    def assertClientCannotReachServer(self, test_client: XdsTestClient):
+        self.assertClientChannelFailed(test_client)
+        self.assertFailedRpcs(test_client)
+
+    def assertClientChannelFailed(self, test_client: XdsTestClient):
         channel = test_client.wait_for_server_channel_state(
             state=_ChannelState.TRANSIENT_FAILURE)
         subchannels = list(
@@ -407,12 +437,6 @@ class SecurityXdsKubernetesTestCase(XdsKubernetesTestCase):
             test_client.channelz.list_subchannels_sockets(subchannels[0]))
         self.assertEmpty(sockets, msg="Client subchannel must have no sockets")
 
-        # With negative tests we can't be absolutely certain expected
-        # failure state is not caused by something else.
-        logger.info(
-            "Client's connectivity state is consistent with a mTLS error "
-            "caused by not presenting mTLS certificate to the server.")
-
     @staticmethod
     def getConnectedSockets(
             test_client: XdsTestClient, test_server: XdsTestServer

tools/run_tests/xds_k8s_test_driver/tests/security_test.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import time
+import uuid
 
 from absl import flags
 from absl.testing import absltest
@@ -21,7 +21,6 @@ from framework import xds_k8s_testcase
 
 logger = logging.getLogger(__name__)
 flags.adopt_module_key_flags(xds_k8s_testcase)
-SKIP_REASON = 'Work in progress'
 
 # Type aliases
 _XdsTestServer = xds_k8s_testcase.XdsTestServer
@@ -48,6 +47,7 @@ class SecurityTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
 
         self.assertTestAppSecurity(_SecurityMode.MTLS, test_client, test_server)
         self.assertSuccessfulRpcs(test_client)
+        logger.info('[SUCCESS] mTLS security mode confirmed.')
 
     def test_tls(self):
         """TLS test.
@@ -66,6 +66,7 @@ class SecurityTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
 
         self.assertTestAppSecurity(_SecurityMode.TLS, test_client, test_server)
         self.assertSuccessfulRpcs(test_client)
+        logger.info('[SUCCESS] TLS security mode confirmed.')
 
     def test_plaintext_fallback(self):
         """Plain-text fallback test.
@@ -86,6 +87,7 @@ class SecurityTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
         self.assertTestAppSecurity(_SecurityMode.PLAINTEXT, test_client,
                                    test_server)
         self.assertSuccessfulRpcs(test_client)
+        logger.info('[SUCCESS] Plaintext security mode confirmed.')
 
     def test_mtls_error(self):
         """Negative test: mTLS Error.
@@ -109,7 +111,8 @@ class SecurityTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
         # Create backend service
         self.td.setup_backend_for_grpc()
 
-        # Start server and attach its NEGs to the backend service
+        # Start server and attach its NEGs to the backend service, but
+        # until they become healthy.
         test_server: _XdsTestServer = self.startSecureTestServer()
         self.setupServerBackends(wait_for_healthy_status=False)
 
@@ -119,38 +122,63 @@ class SecurityTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
                                    client_tls=True,
                                    client_mtls=False)
 
-        # Create the routing rule map
+        # Create the routing rule map.
         self.td.setup_routing_rule_map_for_grpc(self.server_xds_host,
                                                 self.server_xds_port)
-        # Wait for backends healthy after url map is created
+        # Now that TD setup is complete, Backend Service can be populated
+        # with healthy backends (NEGs).
         self.td.wait_for_backends_healthy_status()
 
-        # Start the client.
+        # Start the client, but don't wait for it to report a healthy channel.
         test_client: _XdsTestClient = self.startSecureTestClient(
             test_server, wait_for_active_server_channel=False)
 
-        # With negative tests we can't be absolutely certain expected
-        # failure state is not caused by something else.
-        # To mitigate for this, we repeat the checks a few times in case
-        # the channel eventually stabilizes and RPCs pass.
-        # TODO(sergiitk): use tenacity retryer, move nums to constants
-        wait_sec = 10
-        checks = 3
-        for check in range(1, checks + 1):
-            self.assertMtlsErrorSetup(test_client)
-            self.assertFailedRpcs(test_client)
-            if check != checks:
-                logger.info(
-                    'Check %s successful, waiting %s sec before the next check',
-                    check, wait_sec)
-                time.sleep(wait_sec)
-
-    @absltest.skip(SKIP_REASON)
+        self.assertClientCannotReachServerRepeatedly(test_client)
+        logger.info(
+            "[SUCCESS] Client's connectivity state is consistent with a mTLS "
+            "error caused by not presenting mTLS certificate to the server.")
+
     def test_server_authz_error(self):
         """Negative test: AuthZ error.
 
         Client does not authorize server because of mismatched SAN name.
+        The order of operations is the same as in `test_mtls_error`.
         """
+        # Create backend service
+        self.td.setup_backend_for_grpc()
+
+        # Start server and attach its NEGs to the backend service, but
+        # until they become healthy.
+        test_server: _XdsTestServer = self.startSecureTestServer()
+        self.setupServerBackends(wait_for_healthy_status=False)
+
+        # Regular TLS setup, but with client policy configured using
+        # intentionality incorrect server_namespace.
+        self.td.setup_server_security(server_namespace=self.server_namespace,
+                                      server_name=self.server_name,
+                                      server_port=self.server_port,
+                                      tls=True,
+                                      mtls=False)
+        incorrect_namespace = f'incorrect-namespace-{uuid.uuid4().hex}'
+        self.td.setup_client_security(server_namespace=incorrect_namespace,
+                                      server_name=self.server_name,
+                                      tls=True,
+                                      mtls=False)
+
+        # Create the routing rule map.
+        self.td.setup_routing_rule_map_for_grpc(self.server_xds_host,
+                                                self.server_xds_port)
+        # Now that TD setup is complete, Backend Service can be populated
+        # with healthy backends (NEGs).
+        self.td.wait_for_backends_healthy_status()
+
+        # Start the client, but don't wait for it to report a healthy channel.
+        test_client: _XdsTestClient = self.startSecureTestClient(
+            test_server, wait_for_active_server_channel=False)
+
+        self.assertClientCannotReachServerRepeatedly(test_client)
+        logger.info("[SUCCESS] Client's connectivity state is consistent with "
+                    "AuthZ error caused by server presenting incorrect SAN.")
 
 
 if __name__ == '__main__':