Account for spaces after tags when extracting text

website/common/tests/test_utils.py

@@ -101,6 +101,10 @@ class ExtractTextTestCase(SimpleTestCase):
             extract_text("<p>Paragraph 1</p>\n<p>Paragraph 2</p>"),
             "Paragraph 1 Paragraph 2",
         )
+        self.assertEqual(
+            extract_text("New <a>stuff</a>."),
+            "New stuff.",
+        )
 
     def test_plain_text(self) -> None:
         self.assertEqual(extract_text("Hello there!"), "Hello there!")

website/common/utils.py

@@ -76,10 +76,9 @@ def extract_text(html: str) -> str:
     """
     Get the plain text of some HTML.
     """
-    lines = (
-        text.strip(" \n") for text in BeautifulSoup(html, "lxml").find_all(text=True)
+    return (
+        BeautifulSoup(html.replace("<p", " <p"), "lxml").get_text().replace("\n", " ")
     )
-    return " ".join(line for line in lines if line)
 
 
 def truncate_string(text: str, words: int) -> str: