TextRank

TextRank pydantic-model

Bases: TopWords

Extracts keyterms using the TextRank algorithm.

Config:

• default: validation_config

Fields:

• doc (str | Doc)
• normalize (Optional[Literal['orth', 'lower', 'lemma']])
• include_pos (Optional[str | Collection[str]])
• stopwords (Optional[str | Collection[str]])
• ngrams (Optional[int | Iterable[int]])
• window_size (Optional[int])
• edge_weighting (Optional[str])
• position_bias (Optional[bool])
• candidate_weighting (Optional[Literal['unique', 'frequency']])
• topn (Optional[int | float])
• keyterms (list[tuple[str, float]] | None)

Source code in lexos/topwords/keyterms/textrank.py

class TextRank(TopWords):
    """Extracts keyterms using the TextRank algorithm."""

    doc: str | Doc = Field(..., description="The raw text or spaCy doc to analyze.")
    normalize: Optional[Literal["orth", "lower", "lemma"]] = Field(
        default=None, description="How to normalize tokens for candidate selection."
    )
    include_pos: Optional[str | Collection[str]] = Field(
        default=("NOUN", "PROPN", "ADJ"),
        description="POS tags to include for candidate selection.",
    )
    stopwords: Optional[str | Collection[str]] = Field(
        default=None, description="Custom stopwords to exclude from candidates."
    )
    ngrams: Optional[int | Iterable[int]] = Field(
        default=1,
        description="The ngram range for candidate selection, e.g., 1 for unigrams, (1, 2) for unigrams and bigrams.",
    )
    window_size: Optional[int] = Field(
        2, gt=0, description="The size of the sliding window for co-occurrence."
    )
    edge_weighting: Optional[str] = Field(
        "binary", description='How to weight edges in the graph: "binary" or "count".'
    )
    position_bias: Optional[bool] = Field(
        False,
        description="Whether to bias towards candidates appearing earlier in the text.",
    )
    candidate_weighting: Optional[Literal["unique", "frequency"]] = Field(
        "unique",
        description="How to weight candidates based on their frequency or uniqueness.",
    )
    topn: Optional[int | float] = Field(
        10,
        gt=0,
        description="The number of top keyterms to return (int or float ratio of candidates).",
    )

    keyterms: list[tuple[str, float]] | None = Field(
        default=None,
        description="Extracted keyterms.",
    )

    model_config = validation_config

    def __init__(self, **kwargs) -> None:
        """Initialize the TextRank object and extract keyterms."""
        super().__init__(**kwargs)
        # Extract keyterms and store them in the `keyterms` field
        self.keyterms = textrank(
            doc=self.doc,
            normalize=self.normalize,
            include_pos=self.include_pos,
            stopwords=self.stopwords,
            ngrams=self.ngrams,
            window_size=self.window_size,
            edge_weighting=self.edge_weighting,
            position_bias=self.position_bias,
            candidate_weighting=self.candidate_weighting,
            topn=self.topn,
        )

    def to_dict(self):
        """Return the extracted keyterms as a dictionary."""
        return {
            "keyterms": [
                {"term": term, "score": score} for term, score in (self.keyterms or [])
            ]
        }

    def to_df(self):
        """Return the extracted keyterms as a pandas DataFrame."""
        return pd.DataFrame(getattr(self, "keyterms", []), columns=["term", "score"])

candidate_weighting: Optional[Literal['unique', 'frequency']] = 'unique' pydantic-field

How to weight candidates based on their frequency or uniqueness.

doc: str | Doc pydantic-field

The raw text or spaCy doc to analyze.

edge_weighting: Optional[str] = 'binary' pydantic-field

How to weight edges in the graph: "binary" or "count".

include_pos: Optional[str | Collection[str]] = ('NOUN', 'PROPN', 'ADJ') pydantic-field

POS tags to include for candidate selection.

ngrams: Optional[int | Iterable[int]] = 1 pydantic-field

The ngram range for candidate selection, e.g., 1 for unigrams, (1, 2) for unigrams and bigrams.

normalize: Optional[Literal['orth', 'lower', 'lemma']] = None pydantic-field

How to normalize tokens for candidate selection.

position_bias: Optional[bool] = False pydantic-field

Whether to bias towards candidates appearing earlier in the text.

stopwords: Optional[str | Collection[str]] = None pydantic-field

Custom stopwords to exclude from candidates.

topn: Optional[int | float] = 10 pydantic-field

The number of top keyterms to return (int or float ratio of candidates).

window_size: Optional[int] = 2 pydantic-field

The size of the sliding window for co-occurrence.

__init__(**kwargs) -> None

Initialize the TextRank object and extract keyterms.

Source code in lexos/topwords/keyterms/textrank.py

def __init__(self, **kwargs) -> None:
    """Initialize the TextRank object and extract keyterms."""
    super().__init__(**kwargs)
    # Extract keyterms and store them in the `keyterms` field
    self.keyterms = textrank(
        doc=self.doc,
        normalize=self.normalize,
        include_pos=self.include_pos,
        stopwords=self.stopwords,
        ngrams=self.ngrams,
        window_size=self.window_size,
        edge_weighting=self.edge_weighting,
        position_bias=self.position_bias,
        candidate_weighting=self.candidate_weighting,
        topn=self.topn,
    )

to_df()

Return the extracted keyterms as a pandas DataFrame.

Source code in lexos/topwords/keyterms/textrank.py

def to_df(self):
    """Return the extracted keyterms as a pandas DataFrame."""
    return pd.DataFrame(getattr(self, "keyterms", []), columns=["term", "score"])

to_dict()

Return the extracted keyterms as a dictionary.

Source code in lexos/topwords/keyterms/textrank.py

def to_dict(self):
    """Return the extracted keyterms as a dictionary."""
    return {
        "keyterms": [
            {"term": term, "score": score} for term, score in (self.keyterms or [])
        ]
    }

textrank(doc: Doc | str, *, normalize: Literal['orth', 'lower', 'lemma'] | None = None, include_pos: Optional[str | Collection[str]] = ('NOUN', 'PROPN', 'ADJ'), stopwords: Optional[str | Collection[str]] = None, ngrams: int | Iterable[int] | None = 1, window_size: int = 2, edge_weighting: str = 'binary', position_bias: bool = False, candidate_weighting: Literal['unique', 'frequency'] = 'unique', topn: int | float = 10) -> list[tuple[str, float]]

Extract key terms from a document using the TextRank algorithm, or a variation thereof.

Parameters:

Name	Type	Description	Default
doc	Doc | str	spaCy Doc or plain string from which to extract keyterms.	required
normalize	Literal['orth', 'lower', 'lemma'] | None	If "lemma", lemmatize terms; if "lower", lowercase terms; if None, use the orthographic forms that appear in doc.	None
include_pos	str | Collection[str] | None	One or more POS tags with which to filter for good candidate keyterms. If None, include tokens of all POS tags (which also allows keyterm extraction from docs without POS-tagging.)	('NOUN', 'PROPN', 'ADJ')
stopwords	str | Collection[str] | None	One or more stopwords to filter out. When provided for spaCy Doc inputs, this overrides the model's built-in stopword flags and only filters tokens in this list.	None
ngrams	int | Iterable[int] | None	Candidate n-gram lengths to extract (for example, 1 or (1, 2, 3)). If None, defaults to unigram extraction.	1
window_size	int	Size of sliding window in which term co-occurrences are determined.	2
edge_weighting	str	If "count", the nodes for all co-occurring terms are connected by edges with weight equal to the number of times they co-occurred within a sliding window; if "binary", all such edges have weight = 1.	'binary'
position_bias	bool	If True, bias the PageRank algorithm for weighting nodes in the word graph, such that words appearing earlier and more frequently in doc tend to get larger weights.	False
candidate_weighting	Literal['unique', 'frequency']	Weighting mode for candidate phrase scoring. If "unique", score each unique candidate phrase once (Textacy behaviour). If "frequency", multiply each candidate phrase's score by its observed frequency in the document.	'unique'
topn	int | float	Number of top-ranked terms to return as key terms. If an integer, represents the absolute number; if a float, value must be in the interval (0.0, 1.0], which is converted to an int by int(round(len(candidates) * topn)).	10

Returns:

Type	Description
list[tuple[str, float]]	list[tuple[str, float]]: Sorted list of top topn key terms and their corresponding TextRank ranking scores.

Notes

Example parameter settings for different TextRank variations:

• TextRank: window_size=2, edge_weighting="binary", position_bias=False
• SingleRank: window_size=10, edge_weighting="count", position_bias=False
• PositionRank: window_size=10, edge_weighting="count", position_bias=True

Source code in lexos/topwords/keyterms/textrank.py

def textrank(
    doc: Doc | str,
    *,
    normalize: Literal["orth", "lower", "lemma"] | None = None,
    include_pos: Optional[str | Collection[str]] = ("NOUN", "PROPN", "ADJ"),
    stopwords: Optional[str | Collection[str]] = None,
    ngrams: int | Iterable[int] | None = 1,
    window_size: int = 2,
    edge_weighting: str = "binary",
    position_bias: bool = False,
    candidate_weighting: Literal["unique", "frequency"] = "unique",
    topn: int | float = 10,
) -> list[tuple[str, float]]:
    """Extract key terms from a document using the TextRank algorithm, or a variation thereof.

    Args:
        doc (Doc | str): spaCy `Doc` or plain string from which to extract keyterms.
        normalize (Literal["orth", "lower", "lemma"] | None): If "lemma", lemmatize
            terms; if "lower", lowercase terms; if None, use the orthographic forms
            that appear in `doc`.
        include_pos (str | Collection[str] | None): One or more POS tags with which
            to filter for good candidate keyterms. If `None`, include tokens of all POS
            tags (which also allows keyterm extraction from docs without POS-tagging.)
        stopwords (str | Collection[str] | None): One or more stopwords to filter out.
            When provided for spaCy `Doc` inputs, this overrides the model's built-in
            stopword flags and only filters tokens in this list.
        ngrams (int | Iterable[int] | None): Candidate n-gram lengths to
            extract (for example, `1` or `(1, 2, 3)`). If `None`, defaults to
            unigram extraction.
        window_size (int): Size of sliding window in which term co-occurrences are
            determined.
        edge_weighting (str): If "count", the nodes for
            all co-occurring terms are connected by edges with weight equal to
            the number of times they co-occurred within a sliding window;
            if "binary", all such edges have weight = 1.
        position_bias (bool): If True, bias the PageRank algorithm for weighting
            nodes in the word graph, such that words appearing earlier and more
            frequently in `doc` tend to get larger weights.
        candidate_weighting (Literal["unique", "frequency"]): Weighting mode for
            candidate phrase scoring. If "unique", score each unique candidate phrase
            once (Textacy behaviour). If "frequency", multiply each candidate phrase's
            score by its observed frequency in the document.
        topn (int | float): Number of top-ranked terms to return as key terms.
            If an integer, represents the absolute number; if a float, value
            must be in the interval (0.0, 1.0], which is converted to an int by
            `int(round(len(candidates) * topn))`.

    Returns:
        list[tuple[str, float]]: Sorted list of top `topn` key terms and their
            corresponding TextRank ranking scores.

    Notes:
        Example parameter settings for different TextRank variations:

        - TextRank: `window_size=2, edge_weighting="binary", position_bias=False`
        - SingleRank: `window_size=10, edge_weighting="count", position_bias=False`
        - PositionRank: `window_size=10, edge_weighting="count", position_bias=True`
    """
    include_pos, stopwords, ngrams, topn = _validate_textrank_args(
        include_pos,
        stopwords,
        ngrams,
        candidate_weighting,
        topn,
    )

    # Build aligned term sequences once and reuse across all processing stages.
    terms = _to_term_sequence(doc)
    if not terms:
        return []
    normalized_terms = list(terms_to_strings(terms, normalize))

    word_pos = _build_position_bias(normalized_terms) if position_bias else None

    # Build a graph from all words in doc, then score them
    graph = network.build_cooccurrence_network(
        normalized_terms,
        window_size=window_size,
        edge_weighting=edge_weighting,
    )
    word_scores = network.rank_nodes_by_pagerank(
        graph, weight="weight", personalization=word_pos
    )

    # Generate candidate terms with frequencies in a single streaming pass.
    # Algorithm optimization: frequency-aware candidate extraction.
    candidate_counts = _get_candidate_counts(
        terms, normalized_terms, include_pos, stopwords, ngrams
    )
    topn = _resolve_topn(topn, len(candidate_counts))

    candidate_scores = _score_candidates(
        candidate_counts, word_scores, candidate_weighting
    )
    sorted_candidate_scores = _rank_candidate_scores(candidate_scores, topn)

    return get_filtered_topn_terms(sorted_candidate_scores, topn, match_threshold=0.8)