Sentence Boundary Splitter For Plain Text

Split plain text into sentences while keeping common abbreviations, decimals, and closing quotes attached to the right sentence.

PythonData Integrity

Code

snippet
ABBREVIATIONS = frozenset(
    {
        "mr",
        "mrs",
        "ms",
        "dr",
        "prof",
        "sr",
        "jr",
        "vs",
        "etc",
        "e.g",
        "i.e",
        "fig",
        "no",
        "inc",
        "ltd",
    }
)
CLOSERS = "\"')]}"


def _is_abbreviation(text: str, dot_index: int) -> bool:
    window = text[max(0, dot_index - 16) : dot_index]
    word = window.rsplit(" ", 1)[-1].lower().lstrip("(\"'")
    return word in ABBREVIATIONS or (len(word) == 1 and word.isalpha())


def split_sentences(text: str) -> list[str]:
    normalized = " ".join(text.split())
    if not normalized:
        return []
    sentences: list[str] = []
    start = 0
    index = 0
    while index < len(normalized):
        char = normalized[index]
        if char in ".!?":
            cursor = index + 1
            if cursor < len(normalized) and normalized[cursor] in CLOSERS:
                cursor += 1
            if cursor < len(normalized) and normalized[cursor] == " ":
                if char == "." and _is_abbreviation(normalized, index):
                    index += 1
                    continue
                sentences.append(normalized[start:cursor].strip())
                start = cursor + 1
                index = cursor
        index += 1
    tail = normalized[start:].strip()
    if tail:
        sentences.append(tail)
    return [sentence for sentence in sentences if sentence]


def main() -> None:
    paragraph = (
        "Dr. Smith joined Acme Inc. in 2019. Revenue grew 12.5% that year! "
        "Was it sustainable? The board said \"yes.\" Next year told another story."
    )
    for number, sentence in enumerate(split_sentences(paragraph), start=1):
        print(f"{number}. {sentence}")


if __name__ == "__main__":
    main()