Generate documentation for internal.container. (#7294)

Since our (public) generated APIs document their use of repeated fields, we should probably document its API, too.

This includes some changes to generate_docs.py to explicitly allow named modules, since the repeated field implementation lives in the "internal" tree.

python/docs/generate_docs.py

@@ -67,12 +67,22 @@ DOCS_DIR = pathlib.Path(__file__).parent.resolve()
 PYTHON_DIR = DOCS_DIR.parent
 SOURCE_DIR = PYTHON_DIR / "google" / "protobuf"
 SOURCE_POSIX = SOURCE_DIR.as_posix()
+
+# Modules which are always included:
+INCLUDED_MODULES = (
+  "google.protobuf.internal.containers",
+)
+
+# Packages to ignore, including all modules (unless in INCLUDED_MODULES):
 IGNORED_PACKAGES = (
   "compiler",
+  "docs",
   "internal",
   "pyext",
   "util",
 )
+
+# Ignored module stems in all packages (unless in INCLUDED_MODULES):
 IGNORED_MODULES = (
   "any_test_pb2",
   "api_pb2",
@@ -81,6 +91,7 @@ IGNORED_MODULES = (
   "test_messages_proto3_pb2",
   "test_messages_proto2",
 )
+
 TOC_REGEX = re.compile(
   r"\.\. START REFTOC.*\.\. END REFTOC\.\n",
   flags=re.DOTALL,
@@ -120,20 +131,28 @@ AUTOMODULE_TEMPLATE = """.. DO NOT EDIT, generated by generate_docs.py.
 def find_modules():
   modules = []
   for module_path in SOURCE_DIR.glob("**/*.py"):
-    package_posix = module_path.parent.as_posix()
-    if any(ignored in package_posix for ignored in IGNORED_PACKAGES):
+    # Determine the (dotted) relative package and module names.
+    package_path = module_path.parent.relative_to(PYTHON_DIR)
+    if package_path == SOURCE_DIR:
+      package_name = ""
+      module_name = module_path.stem
+    else:
+      package_name = package_path.as_posix().replace("/", ".")
+      module_name = package_name + "." + module_path.stem
+
+    # Filter: first, accept anything in the whitelist; then, reject anything
+    # at package level, then module name level.
+    if any(include == module_name for include in INCLUDED_MODULES):
+      pass
+    elif any(ignored in package_name for ignored in IGNORED_PACKAGES):
       continue
-    if any(ignored in module_path.stem for ignored in IGNORED_MODULES):
+    elif any(ignored in module_path.stem for ignored in IGNORED_MODULES):
       continue
 
-    package_name = "google.protobuf{}".format(
-      package_posix[len(SOURCE_POSIX) :].replace("/", ".")
-    )
     if module_path.name == "__init__.py":
       modules.append(package_name)
     else:
-      module_name = module_path.stem
-      modules.append("{}.{}".format(package_name, module_name))
+      modules.append(module_name)
 
   return modules
 

python/docs/google/protobuf/internal/containers.rst

@@ -0,0 +1,21 @@
+.. DO NOT EDIT, generated by generate_docs.py.
+
+.. ifconfig:: build_env == 'readthedocs'
+
+   .. warning::
+
+      You are reading the documentation for the `latest committed changes
+      <https://github.com/protocolbuffers/protobuf/tree/master/python>`_ of
+      the `Protocol Buffers package for Python
+      <https://developers.google.com/protocol-buffers/docs/pythontutorial>`_.
+      Some features may not yet be released. Read the documentation for the
+      latest released package at `googleapis.dev
+      <https://googleapis.dev/python/protobuf/latest/>`_.
+
+google.protobuf.internal.containers
+===================================
+
+.. automodule:: google.protobuf.internal.containers
+   :members:
+   :inherited-members:
+   :undoc-members:

python/docs/index.rst

@@ -38,6 +38,7 @@ Modules and Packages
    google/protobuf/duration_pb2
    google/protobuf/empty_pb2
    google/protobuf/field_mask_pb2
+   google/protobuf/internal/containers
    google/protobuf/json_format
    google/protobuf/message
    google/protobuf/message_factory