Sentence Boundary Splitter For Plain Text
Split plain text into sentences while keeping common abbreviations, decimals, and closing quotes attached to the right sentence.
Code
ABBREVIATIONS = frozenset(
{
"mr",
"mrs",
"ms",
"dr",
"prof",
"sr",
"jr",
"vs",
"etc",
"e.g",
"i.e",
"fig",
"no",
"inc",
"ltd",
}
)
CLOSERS = "\"')]}"
def _is_abbreviation(text: str, dot_index: int) -> bool:
window = text[max(0, dot_index - 16) : dot_index]
word = window.rsplit(" ", 1)[-1].lower().lstrip("(\"'")
return word in ABBREVIATIONS or (len(word) == 1 and word.isalpha())
def split_sentences(text: str) -> list[str]:
normalized = " ".join(text.split())
if not normalized:
return []
sentences: list[str] = []
start = 0
index = 0
while index < len(normalized):
char = normalized[index]
if char in ".!?":
cursor = index + 1
if cursor < len(normalized) and normalized[cursor] in CLOSERS:
cursor += 1
if cursor < len(normalized) and normalized[cursor] == " ":
if char == "." and _is_abbreviation(normalized, index):
index += 1
continue
sentences.append(normalized[start:cursor].strip())
start = cursor + 1
index = cursor
index += 1
tail = normalized[start:].strip()
if tail:
sentences.append(tail)
return [sentence for sentence in sentences if sentence]
def main() -> None:
paragraph = (
"Dr. Smith joined Acme Inc. in 2019. Revenue grew 12.5% that year! "
"Was it sustainable? The board said \"yes.\" Next year told another story."
)
for number, sentence in enumerate(split_sentences(paragraph), start=1):
print(f"{number}. {sentence}")
if __name__ == "__main__":
main()Report a problem with this exampleOpens a pre-filled issue on GitHub.