Reduce test requirements duplication, sync flake8 and related (#28538)

* Generate pre-commit test dependencies instead of duplicating

* Upgrade/sync to flake8 3.7.9, flake8-docstrings 1.5.0, and pydocstyle 4.0.1

https://flake8.readthedocs.io/en/latest/release-notes/3.7.9.html
https://gitlab.com/pycqa/flake8-docstrings/blob/1.4.0/HISTORY.rst
https://gitlab.com/pycqa/flake8-docstrings/blob/1.5.0/HISTORY.rst
http://www.pydocstyle.org/en/4.0.1/release_notes.html

* Include requirements_test.txt from *_all.txt instead of copying

script/gen_requirements_all.py

@@ -8,6 +8,8 @@ import pkgutil
 import re
 import sys
 
+from homeassistant.util.yaml.loader import load_yaml
+
 from script.hassfest.model import Integration
 
 COMMENT_REQUIREMENTS = (
@@ -225,10 +227,11 @@ def requirements_all_output(reqs):
 def requirements_test_output(reqs):
     """Generate output for test_requirements."""
     output = []
-    output.append("# Home Assistant test")
-    output.append("\n")
-    output.append(Path("requirements_test.txt").read_text())
-    output.append("\n")
+    output.append("# Home Assistant tests, full dependency set\n")
+    output.append(
+        f"# Automatically generated by {Path(__file__).name}, do not edit\n\n"
+    )
+    output.append("-r requirements_test.txt\n")
 
     filtered = {
         requirement: modules
@@ -246,6 +249,24 @@ def requirements_test_output(reqs):
     return "".join(output)
 
 
+def requirements_pre_commit_output():
+    """Generate output for pre-commit dependencies."""
+    source = ".pre-commit-config-all.yaml"
+    pre_commit_conf = load_yaml(source)
+    reqs = []
+    for repo in (x for x in pre_commit_conf["repos"] if x.get("rev")):
+        for hook in repo["hooks"]:
+            reqs.append(f"{hook['id']}=={repo['rev']}")
+            reqs.extend(x for x in hook.get("additional_dependencies", ()))
+    output = [
+        f"# Automatically generated "
+        f"from {source} by {Path(__file__).name}, do not edit",
+        "",
+    ]
+    output.extend(sorted(reqs))
+    return "\n".join(output) + "\n"
+
+
 def gather_constraints():
     """Construct output for constraint file."""
     return (
@@ -285,10 +306,12 @@ def main(validate):
 
     reqs_file = requirements_all_output(data)
     reqs_test_file = requirements_test_output(data)
+    reqs_pre_commit_file = requirements_pre_commit_output()
     constraints = gather_constraints()
 
     files = (
         ("requirements_all.txt", reqs_file),
+        ("requirements_test_pre_commit.txt", reqs_pre_commit_file),
         ("requirements_test_all.txt", reqs_test_file),
         ("homeassistant/package_constraints.txt", constraints),
     )