sCAKE

SCake pydantic-model

Bases: TopWords

Extracts keyterms using the sCAKE (Semantic Connectivity Aware Keyword Extraction) algorithm.

Config:

• default: validation_config

Fields:

• doc (str | Doc)
• normalize (Optional[Literal['orth', 'lower', 'lemma']])
• include_pos (Optional[str | Collection[str]])
• topn (Optional[int | float])
• keyterms (list[tuple[str, float]] | None)

Source code in lexos/topwords/keyterms/scake.py

class SCake(TopWords):
    """Extracts keyterms using the sCAKE (Semantic Connectivity Aware Keyword Extraction) algorithm."""

    doc: str | Doc = Field(..., description="The raw text or spaCy doc to analyze.")
    normalize: Optional[Literal["orth", "lower", "lemma"]] = Field(
        default="lemma",
        description="How to normalize tokens for candidate selection.",
    )

    include_pos: Optional[str | Collection[str]] = Field(
        default=("NOUN", "PROPN", "ADJ"),
        description="POS tags to include for candidate selection.",
    )

    topn: Optional[int | float] = Field(
        10,
        gt=0,
        description="The number of top keyterms to return (int or float ratio of candidates).",
    )

    keyterms: list[tuple[str, float]] | None = Field(
        default=None,
        description="Extracted keyterms.",
    )

    model_config = validation_config

    def __init__(self, **kwargs) -> None:
        """Initialize the SCake object and extract keyterms."""
        super().__init__(**kwargs)
        # Extract keyterms and put them in field
        self.keyterms = scake(
            doc=self.doc,
            normalize=self.normalize,
            include_pos=self.include_pos,
            topn=self.topn,
        )

    def to_dict(self):
        """Return the extracted keyterms as a dictionary."""
        return {
            "keyterms": [
                {"term": term, "score": score} for term, score in (self.keyterms or [])
            ]
        }

    def to_df(self):
        """Return the extracted keyterms as a pandas DataFrame."""
        return pd.DataFrame(getattr(self, "keyterms", []), columns=["term", "score"])

doc: str | Doc pydantic-field

The raw text or spaCy doc to analyze.

include_pos: Optional[str | Collection[str]] = ('NOUN', 'PROPN', 'ADJ') pydantic-field

POS tags to include for candidate selection.

normalize: Optional[Literal['orth', 'lower', 'lemma']] = 'lemma' pydantic-field

How to normalize tokens for candidate selection.

topn: Optional[int | float] = 10 pydantic-field

The number of top keyterms to return (int or float ratio of candidates).

__init__(**kwargs) -> None

Initialize the SCake object and extract keyterms.

Source code in lexos/topwords/keyterms/scake.py

def __init__(self, **kwargs) -> None:
    """Initialize the SCake object and extract keyterms."""
    super().__init__(**kwargs)
    # Extract keyterms and put them in field
    self.keyterms = scake(
        doc=self.doc,
        normalize=self.normalize,
        include_pos=self.include_pos,
        topn=self.topn,
    )

to_df()

Return the extracted keyterms as a pandas DataFrame.

Source code in lexos/topwords/keyterms/scake.py

def to_df(self):
    """Return the extracted keyterms as a pandas DataFrame."""
    return pd.DataFrame(getattr(self, "keyterms", []), columns=["term", "score"])

to_dict()

Return the extracted keyterms as a dictionary.

Source code in lexos/topwords/keyterms/scake.py

def to_dict(self):
    """Return the extracted keyterms as a dictionary."""
    return {
        "keyterms": [
            {"term": term, "score": score} for term, score in (self.keyterms or [])
        ]
    }

scake(doc: Doc | str, *, normalize: Literal['orth', 'lower', 'lemma'] | None = 'lemma', include_pos: Optional[str | Collection[str]] = ('NOUN', 'PROPN', 'ADJ'), topn: int | float = 10) -> list[tuple[str, float]]

Extract key terms from a document using the sCAKE algorithm.

Parameters:

Name	Type	Description	Default
doc	Doc | str	spaCy Doc or plain string from which to extract keyterms. If a Doc and it has sentence boundaries, they will be used.	required
normalize	Literal['orth', 'lower', 'lemma'] | None	If "lemma", lemmatize terms; if "lower", lowercase terms; if None, use the orthographic forms that appear in doc; if a callable, must accept a Token and return a str.	'lemma'
include_pos	str | Collection[str] | None	One or more POS tags with which to filter for good candidate keyterms. If None, include tokens of all POS tags (which also allows keyterm extraction from docs without POS-tagging.)	('NOUN', 'PROPN', 'ADJ')
topn	int | float	Number of top-ranked terms to return as key terms. If an integer, represents the absolute number; if a float, value must be in the interval (0.0, 1.0], which is converted to an int by int(round(len(candidates) * topn)).	10

Returns:

Type	Description
list[tuple[str, float]]	list[tuple[str, float]]: Sorted list of top topn key terms and their corresponding sCAKE scores.

Notes

1. Normalize the arguments
2. Then make the co-occurrence matrix.
3. Make a graph based on the matrix.
4. Compute the scores based on the graph.
5. Get the key phrases and return topn.

Source code in lexos/topwords/keyterms/scake.py

def scake(
    doc: Doc | str,
    *,
    normalize: Literal["orth", "lower", "lemma"] | None = "lemma",
    include_pos: Optional[str | Collection[str]] = ("NOUN", "PROPN", "ADJ"),
    topn: int | float = 10,
) -> list[tuple[str, float]]:
    """Extract key terms from a document using the sCAKE algorithm.

    Args:
        doc (Doc | str): spaCy `Doc` or plain string from which to extract keyterms.
            If a `Doc` and it has sentence boundaries, they will be used.
        normalize (Literal["orth", "lower", "lemma"] | None): If "lemma", lemmatize
            terms; if "lower", lowercase terms; if None, use the orthographic forms
            that appear in `doc`; if a callable, must accept a `Token` and return a str.
        include_pos (str | Collection[str] | None): One or more POS tags with which
            to filter for good candidate keyterms. If `None`, include tokens of all POS
            tags (which also allows keyterm extraction from docs without POS-tagging.)
        topn (int | float): Number of top-ranked terms to return as key terms.
            If an integer, represents the absolute number; if a float, value
            must be in the interval (0.0, 1.0], which is converted to an int by
            `int(round(len(candidates) * topn))`.

    Returns:
        list[tuple[str, float]]: Sorted list of top `topn` key terms and their
            corresponding sCAKE scores.

    Notes:
        1. Normalize the arguments
        2. Then make the co-occurrence matrix.
        3. Make a graph based on the matrix.
        4. Compute the scores based on the graph.
        5. Get the key phrases and return topn.
    """
    include_pos_set, topn = _validate_scake_args(include_pos, topn)

    # Convert str input to term sequence
    terms = _to_term_sequence(doc)
    if not terms:
        return []

    # Build normalized str sequence aligned with terms
    normalized_terms = list(terms_to_strings(terms, normalize))

    # Build co-occurrence matrix over sentence segments(or the whole doc if plain-string input, which has no sentence boundaries)
    cooc_mat = _build_cooc_matrix(doc, terms, normalized_terms, include_pos_set)
    if not cooc_mat:
        return []

    # Build the word graph from the co-occurrence matrix
    graph = nx.Graph()
    graph.add_edges_from(
        (w1, w2, {"weight": weight}) for (w1, w2), weight in cooc_mat.items()
    )

    # Compute the scores
    word_scores = _compute_word_scores(terms, normalized_terms, graph, cooc_mat)
    if not word_scores:
        return []

    # Get candidate phrases and resolve topn
    candidates = _get_candidates(terms, normalized_terms, include_pos_set)
    topn = _resolve_topn(topn, len(candidates))

    candidate_scores = {}
    for candidate in candidates:
        phrase = " ".join(candidate)
        score = sum(word_scores.get(word, 0.0) for word in candidate)
        candidate_scores[phrase] = score
        sorted_candidate_scores = sorted(
            candidate_scores.items(), key=itemgetter(1, 0), reverse=True
        )

    return get_filtered_topn_terms(sorted_candidate_scores, topn, match_threshold=0.8)