Fix readthedocs builds (#3046)

- Mitigate removal of the -E build option (rtd now reuses build env)
- Fix psycopg module's docs
- Properly remove modules/* docs for epub and latex

---------

Co-authored-by: Alexander Kukushkin <[email protected]>

docs/conf.py

@@ -21,6 +21,8 @@ import os
 
 import sys
 
+from sphinx.application import ENV_PICKLE_FILENAME
+
 sys.path.insert(0, os.path.abspath('..'))
 
 from patroni.version import __version__
@@ -243,13 +245,24 @@ intersphinx_mapping = {'python': ('https://docs.python.org/', None)}
 # Remove these pages from index, references, toc trees, etc.
 # If the builder is not 'html' then add the API docs modules index to pages to be removed.
 exclude_from_builder = {
-    'latex': ['modules/modules'],
-    'epub': ['modules/modules'],
+    'latex': ['modules/'],
+    'epub': ['modules/'],
 }
 # Internal holding list, anything added here will always be excluded
 _docs_to_remove = []
 
 
+def config_inited(app, config):
+    """Run during Sphinx `config-inited` phase.
+
+    rtd reuses the environment, and there is no way to customize this behavior.
+    Thus we remove the saved env.
+    """
+    pickle_file = os.path.join(app.doctreedir, ENV_PICKLE_FILENAME)
+    if on_rtd and os.path.exists(pickle_file):
+        os.remove(pickle_file)
+
+
 def builder_inited(app):
     """Run during Sphinx `builder-inited` phase.
 
@@ -263,14 +276,26 @@ def builder_inited(app):
         _docs_to_remove.extend(exclude_from_builder[app.builder.name])
 
 
+def _to_be_removed(doc):
+    for remove in _docs_to_remove:
+        if doc.startswith(remove):
+            return True
+    return False
+
+
 def env_get_outdated(app, env, added, changed, removed):
     """Run during Sphinx `env-get-outdated` phase.
 
     Remove the items listed in `docs_to_remove` from known pages.
     """
-    added.difference_update(_docs_to_remove)
-    changed.difference_update(_docs_to_remove)
-    removed.update(_docs_to_remove)
+    to_remove = set()
+    for doc in env.found_docs:
+        if _to_be_removed(doc):
+            to_remove.add(doc)
+    added.difference_update(to_remove)
+    changed.difference_update(to_remove)
+    removed.update(to_remove)
+    env.project.docnames.difference_update(to_remove)
     return []
 
 
@@ -282,8 +307,7 @@ def doctree_read(app, doctree):
     from sphinx import addnodes
     for toc_tree_node in doctree.traverse(addnodes.toctree):
         for e in toc_tree_node['entries']:
-            ref = str(e[1])
-            if ref in _docs_to_remove:
+            if _to_be_removed(str(e[1])):
                 toc_tree_node['entries'].remove(e)
 
 
@@ -304,6 +328,7 @@ def setup(app):
         app.add_stylesheet('custom.css')
 
     # Run extra steps to remove module docs when running with a non-html builder
+    app.connect('config-inited', config_inited)
     app.connect('builder-inited', builder_inited)
     app.connect('env-get-outdated', env_get_outdated)
     app.connect('doctree-read', doctree_read)