YAKE

Yake pydantic-model

Bases: TopWords

Extracts keyterms using the YAKE algorithm.

Config:

• default: validation_config

Fields:

• doc (DocLike)
• normalize (Optional[Literal['orth', 'lower', 'lemma', 'norm'] | None])
• include_pos (Optional[str | Collection[str]])
• stopwords (Optional[str | Collection[str]])
• ngrams (Optional[int | Iterable[int] | None])
• window_size (Optional[int])
• topn (Optional[int | float])
• keyterms (list[tuple[str, float]] | None)

Source code in lexos/topwords/keyterms/yake.py

class Yake(TopWords):
    """Extracts keyterms using the YAKE algorithm."""

    doc: DocLike = Field(
        ...,
        description="Input as a spaCy doc, raw string, or sequence of terms.",
    )
    normalize: Optional[Literal["orth", "lower", "lemma", "norm"] | None] = Field(
        default="lemma", description="How to normalize terms for scoring."
    )
    include_pos: Optional[str | Collection[str]] = Field(
        default=("NOUN", "PROPN", "ADJ"),
        description="POS tags to include for candidates; ignored when unavailable.",
    )
    stopwords: Optional[str | Collection[str]] = Field(
        default=None,
        description="Custom stopwords to exclude from candidates and scoring.",
    )
    ngrams: Optional[int | Iterable[int] | None] = Field(
        default=(1, 2, 3),
        description="N-gram sizes to consider for keyterm candidates.",
    )
    window_size: Optional[int] = Field(
        2,
        gt=0,
        description="Context window size on each side of each term.",
    )
    topn: Optional[int | float] = Field(
        10,
        gt=0,
        description="Number of top keyterms (or ratio if float in (0, 1]).",
    )

    keyterms: list[tuple[str, float]] | None = Field(
        default=None,
        description="Extracted keyterms as (term, score) tuples.",
    )

    model_config = validation_config

    def __init__(self, **kwargs):
        """Initialize the Yake object and extract keyterms."""
        super().__init__(**kwargs)

        self.keyterms = yake(
            doc=self.doc,
            normalize=self.normalize,
            include_pos=self.include_pos,
            stopwords=self.stopwords,
            ngrams=self.ngrams,
            window_size=self.window_size,
            topn=self.topn,
        )

    def to_dict(self) -> dict[str, Any]:
        """Return the extracted keyterms as a dictionary.

        Returns:
            dict[str, Any]: A dictionary containing the extracted keyterms.
        """
        return {
            "keyterms": [
                {"term": term, "score": score} for term, score in (self.keyterms or [])
            ]
        }

    def to_df(self) -> pd.DataFrame:
        """Return the extracted keyterms as a pandas DataFrame.

        Returns:
            pd.DataFrame: A DataFrame with columns 'term' and 'score' containing the extracted keyterms.
        """
        return pd.DataFrame(getattr(self, "keyterms", []), columns=["term", "score"])

doc: DocLike pydantic-field

Input as a spaCy doc, raw string, or sequence of terms.

include_pos: Optional[str | Collection[str]] = ('NOUN', 'PROPN', 'ADJ') pydantic-field

POS tags to include for candidates; ignored when unavailable.

ngrams: Optional[int | Iterable[int] | None] = (1, 2, 3) pydantic-field

N-gram sizes to consider for keyterm candidates.

normalize: Optional[Literal['orth', 'lower', 'lemma', 'norm'] | None] = 'lemma' pydantic-field

How to normalize terms for scoring.

stopwords: Optional[str | Collection[str]] = None pydantic-field

Custom stopwords to exclude from candidates and scoring.

topn: Optional[int | float] = 10 pydantic-field

Number of top keyterms (or ratio if float in (0, 1]).

window_size: Optional[int] = 2 pydantic-field

Context window size on each side of each term.

__init__(**kwargs)

Initialize the Yake object and extract keyterms.

Source code in lexos/topwords/keyterms/yake.py

def __init__(self, **kwargs):
    """Initialize the Yake object and extract keyterms."""
    super().__init__(**kwargs)

    self.keyterms = yake(
        doc=self.doc,
        normalize=self.normalize,
        include_pos=self.include_pos,
        stopwords=self.stopwords,
        ngrams=self.ngrams,
        window_size=self.window_size,
        topn=self.topn,
    )

to_df() -> pd.DataFrame

Return the extracted keyterms as a pandas DataFrame.

Returns:

Type	Description
DataFrame	pd.DataFrame: A DataFrame with columns 'term' and 'score' containing the extracted keyterms.

Source code in lexos/topwords/keyterms/yake.py

def to_df(self) -> pd.DataFrame:
    """Return the extracted keyterms as a pandas DataFrame.

    Returns:
        pd.DataFrame: A DataFrame with columns 'term' and 'score' containing the extracted keyterms.
    """
    return pd.DataFrame(getattr(self, "keyterms", []), columns=["term", "score"])

to_dict() -> dict[str, Any]

Return the extracted keyterms as a dictionary.

Returns:

Type	Description
dict[str, Any]	dict[str, Any]: A dictionary containing the extracted keyterms.

Source code in lexos/topwords/keyterms/yake.py

def to_dict(self) -> dict[str, Any]:
    """Return the extracted keyterms as a dictionary.

    Returns:
        dict[str, Any]: A dictionary containing the extracted keyterms.
    """
    return {
        "keyterms": [
            {"term": term, "score": score} for term, score in (self.keyterms or [])
        ]
    }

yake(doc: DocLike, *, normalize: Literal['orth', 'lower', 'lemma', 'norm'] | None = 'lemma', include_pos: Optional[str | Collection[str]] = ('NOUN', 'PROPN', 'ADJ'), stopwords: Optional[str | Collection[str]] = None, ngrams: int | Iterable[int] | None = (1, 2, 3), window_size: int = 2, topn: int | float = 10) -> list[tuple[str, float]]

Extract key terms from a document using the YAKE algorithm.

This implementation is inspired by Textacy's YAKE extractor but adds compatibility for raw strings and sequences of strings/token-like objects.

Parameters:

Name	Type	Description	Default
doc	DocLike	Input document as a spaCy Doc, raw string, or sequence of terms.	required
normalize	Literal['orth', 'lower', 'lemma', 'norm'] | None	How to normalize terms for scoring.	'lemma'
include_pos	Optional[str | Collection[str]]	POS tags to include for candidates; ignored when unavailable.	('NOUN', 'PROPN', 'ADJ')
stopwords	Optional[str | Collection[str]]	Custom stopwords to exclude from candidates and scoring.	None
ngrams	int | Iterable[int] | None	N-gram sizes to consider for keyterm candidates.	(1, 2, 3)
window_size	int	Context window size on each side of each term.	2
topn	int | float	Number of top keyterms to return (or ratio if float in (0, 1]).	10

Returns:

Type	Description
list[tuple[str, float]]	list[tuple[str, float]]: Extracted keyterms as (term, score) tuples.

Source code in lexos/topwords/keyterms/yake.py

def yake(
    doc: DocLike,
    *,
    normalize: Literal["orth", "lower", "lemma", "norm"] | None = "lemma",
    include_pos: Optional[str | Collection[str]] = ("NOUN", "PROPN", "ADJ"),
    stopwords: Optional[str | Collection[str]] = None,
    ngrams: int | Iterable[int] | None = (1, 2, 3),
    window_size: int = 2,
    topn: int | float = 10,
) -> list[tuple[str, float]]:
    """Extract key terms from a document using the YAKE algorithm.

    This implementation is inspired by Textacy's YAKE extractor but adds
    compatibility for raw strings and sequences of strings/token-like objects.

    Args:
        doc (DocLike): Input document as a spaCy Doc, raw string, or sequence of terms.
        normalize (Literal["orth", "lower", "lemma", "norm"] | None): How to normalize terms for scoring.
        include_pos (Optional[str | Collection[str]]): POS tags to include for candidates; ignored when unavailable.
        stopwords (Optional[str | Collection[str]]): Custom stopwords to exclude from candidates and scoring.
        ngrams (int | Iterable[int] | None): N-gram sizes to consider for keyterm candidates.
        window_size (int): Context window size on each side of each term.
        topn (int | float): Number of top keyterms to return (or ratio if float in (0, 1]).

    Returns:
        list[tuple[str, float]]: Extracted keyterms as (term, score) tuples.
    """
    include_pos_set, stopwords_set, ngram_sizes, topn = _validate_yake_args(
        include_pos=include_pos,
        stopwords=stopwords,
        ngrams=ngrams,
        normalize=normalize,
        topn=topn,
    )
    terms, sent_ids = _to_terms_and_sentence_ids(doc)
    if not terms:
        return []

    stop_words: set[str] = set()
    seen_candidates: set[str] = set()

    word_occ_vals = _get_per_word_occurrence_values(
        terms=terms,
        sent_ids=sent_ids,
        normalize=normalize,
        stop_words=stop_words,
        window_size=window_size,
        custom_stopwords=stopwords_set,
    )
    if not word_occ_vals:
        return []

    word_freqs = {w_id: len(vals["is_uc"]) for w_id, vals in word_occ_vals.items()}
    word_scores = _compute_word_scores(
        sent_ids=sent_ids,
        word_occ_vals=word_occ_vals,
        word_freqs=word_freqs,
        stop_words=stop_words,
    )

    term_scores: dict[str, float] = {}

    if 1 in ngram_sizes:
        candidates = _get_unigram_candidates(
            terms,
            include_pos=include_pos_set,
            custom_stopwords=stopwords_set,
        )
        _score_unigram_candidates(
            candidates=candidates,
            word_freqs=word_freqs,
            word_scores=word_scores,
            term_scores=term_scores,
            stop_words=stop_words,
            seen_candidates=seen_candidates,
            normalize=normalize,
        )

    ngram_candidates = _get_ngram_candidates(
        terms,
        sent_ids,
        n_sizes=tuple(n for n in ngram_sizes if n > 1),
        include_pos=include_pos_set,
        custom_stopwords=stopwords_set,
    )
    ngram_freqs = collections.Counter(
        " ".join(_term_to_str(term, normalize) for term in ngram)
        for ngram in ngram_candidates
    )
    _score_ngram_candidates(
        candidates=ngram_candidates,
        ngram_freqs=ngram_freqs,
        word_scores=word_scores,
        term_scores=term_scores,
        seen_candidates=seen_candidates,
        normalize=normalize,
    )

    if isinstance(topn, float):
        topn = int(round(len(seen_candidates) * topn))

    sorted_term_scores = sorted(term_scores.items(), key=lambda item: item[1])
    return get_filtered_topn_terms(sorted_term_scores, topn, match_threshold=0.8)