Keyterms Utilities

terms_to_strings(terms: Iterable[Span | Token | str], by: str | Callable[[Span | Token | str], str] | None) -> Iterable[str]

Transform a sequence of terms as spaCy Tokens, Spans, or strings into strings.

Parameters:

Name	Type	Description	Default
terms	Iterable[Span | Token | str]	Terms to transform into strings.	required
by	str | Callable[[Span | Token | str], str]	Method by which terms are transformed into strings. If "orth" or None, terms are represented by their text exactly as written; if "lower", by the lowercased form of their text; if "lemma", by their base form w/o inflectional suffixes; if a callable, must accept a Token, Span, or str and return a string.	required

Yields:

Type	Description
Iterable[str]	Iterable[str]: Next term in terms, as a string.

Source code in lexos/topwords/keyterms/keyterms_util.py

def terms_to_strings(
    terms: Iterable[Span | Token | str],
    by: str | Callable[[Span | Token | str], str] | None,
) -> Iterable[str]:
    """Transform a sequence of terms as spaCy Tokens, Spans, or strings into strings.

    Args:
        terms (Iterable[Span | Token | str]): Terms to transform into strings.
        by (str | Callable[[Span | Token | str], str]): Method by which terms are transformed into strings.
            If "orth" or None, terms are represented by their text exactly as written;
            if "lower", by the lowercased form of their text;
            if "lemma", by their base form w/o inflectional suffixes;
            if a callable, must accept a `Token`, `Span`, or `str` and return a string.

    Yields:
        Iterable[str]: Next term in `terms`, as a string.
    """
    terms_: Iterable[str]
    if by in ("orth", None):
        terms_ = (
            term.text if isinstance(term, (Token, Span)) else term for term in terms
        )
    elif by == "lower":
        terms_ = (
            term.lower() if isinstance(term, str) else term.text.lower()
            for term in terms
        )
    elif by == "lemma":
        terms_ = (term.lemma_ if isinstance(term, Token) else term for term in terms)
    elif callable(by):
        terms_ = (by(term) for term in terms)
    else:
        raise ValueError(
            f"by={by} is invalid; must be one of {{'orth', 'lower', 'lemma', Callable}}"
        )
    for term in terms_:
        yield term

is_unicode_punctuation(ch: str) -> bool

Return True if ch is a Unicode punctuation character.

Parameters:

Name	Type	Description	Default
ch	str	Single character to check.	required

Returns:

Name	Type	Description
bool	bool	True if ch is a Unicode punctuation character, False otherwise.

Source code in lexos/topwords/keyterms/keyterms_util.py

def is_unicode_punctuation(ch: str) -> bool:
    """Return True if `ch` is a Unicode punctuation character.

    Args:
        ch (str): Single character to check.

    Returns:
        bool: True if `ch` is a Unicode punctuation character, False otherwise.
    """
    # 'P' stands for Punctuation categories (Pc, Pd, Pe, Pf, Pi, Po, Ps)
    return bool(ch) and all(unicodedata.category(char).startswith("P") for char in ch)