SGRank

SGRank pydantic-model

Bases: TopWords

Extracts keyterms using the SGRank algorithm.

Config:

• default: validation_config

Fields:

• doc (str | Doc)
• normalize (Optional[str | Callable[[Span], str]])
• ngrams (int | Collection[int])
• include_pos (Optional[str | Collection[str]])
• window_size (Optional[int])
• topn (Optional[int | float])
• idf (Optional[dict[str, float]])
• keyterms (list[tuple[str, float]] | None)

Source code in lexos/topwords/keyterms/sgrank.py

class SGRank(TopWords):
    """Extracts keyterms using the SGRank algorithm."""

    doc: str | Doc = Field(..., description="The raw text or spaCy doc to analyze.")
    normalize: Optional[str | Callable[[Span], str]] = Field(
        default="lemma", description="How to normalize candidates for scoring."
    )
    ngrams: int | Collection[int] = Field(
        default=(1, 2, 3, 4, 5, 6),
        description="N-gram sizes to consider for keyterm candidates.",
    )
    include_pos: Optional[str | Collection[str]] = Field(
        default=("NOUN", "PROPN", "ADJ"),
        description="POS tags to include for candidate selection.",
    )
    window_size: Optional[int] = Field(
        1500,
        gt=1,
        description="Size of the sliding window for co-occurrence, in tokens.",
    )
    topn: Optional[int | float] = Field(
        10,
        gt=0,
        description="The number of top keyterms to return (int or float ratio of candidates).",
    )
    idf: Optional[dict[str, float]] = Field(
        default=None,
        description="Mapping of normalized term to inverse document frequency.",
    )

    keyterms: list[tuple[str, float]] | None = Field(
        default=None,
        description="Extracted keyterms.",
    )

    model_config = validation_config

    def __init__(self, **kwargs) -> None:
        """Initialize the SGRank object and extract keyterms."""
        super().__init__(**kwargs)
        self.keyterms = sgrank(
            doc=self.doc,
            normalize=self.normalize,
            ngrams=self.ngrams,
            include_pos=self.include_pos,
            window_size=self.window_size,
            topn=self.topn,
            idf=self.idf,
        )

    def to_dict(self):
        """Return the extracted keyterms as a dictionary."""
        return {
            "keyterms": [
                {"term": term, "score": score} for term, score in (self.keyterms or [])
            ]
        }

    def to_df(self):
        """Return the extracted keyterms as a pandas DataFrame."""
        return pd.DataFrame(getattr(self, "keyterms", []), columns=["term", "score"])

doc: str | Doc pydantic-field

The raw text or spaCy doc to analyze.

idf: Optional[dict[str, float]] = None pydantic-field

Mapping of normalized term to inverse document frequency.

include_pos: Optional[str | Collection[str]] = ('NOUN', 'PROPN', 'ADJ') pydantic-field

POS tags to include for candidate selection.

ngrams: int | Collection[int] = (1, 2, 3, 4, 5, 6) pydantic-field

N-gram sizes to consider for keyterm candidates.

normalize: Optional[str | Callable[[Span], str]] = 'lemma' pydantic-field

How to normalize candidates for scoring.

topn: Optional[int | float] = 10 pydantic-field

The number of top keyterms to return (int or float ratio of candidates).

window_size: Optional[int] = 1500 pydantic-field

Size of the sliding window for co-occurrence, in tokens.

__init__(**kwargs) -> None

Initialize the SGRank object and extract keyterms.

Source code in lexos/topwords/keyterms/sgrank.py

def __init__(self, **kwargs) -> None:
    """Initialize the SGRank object and extract keyterms."""
    super().__init__(**kwargs)
    self.keyterms = sgrank(
        doc=self.doc,
        normalize=self.normalize,
        ngrams=self.ngrams,
        include_pos=self.include_pos,
        window_size=self.window_size,
        topn=self.topn,
        idf=self.idf,
    )

to_df()

Return the extracted keyterms as a pandas DataFrame.

Source code in lexos/topwords/keyterms/sgrank.py

def to_df(self):
    """Return the extracted keyterms as a pandas DataFrame."""
    return pd.DataFrame(getattr(self, "keyterms", []), columns=["term", "score"])

to_dict()

Return the extracted keyterms as a dictionary.

Source code in lexos/topwords/keyterms/sgrank.py

def to_dict(self):
    """Return the extracted keyterms as a dictionary."""
    return {
        "keyterms": [
            {"term": term, "score": score} for term, score in (self.keyterms or [])
        ]
    }

sgrank(doc: Doc | str, *, normalize: Optional[str | Callable[[Span], str]] = 'lemma', ngrams: int | Collection[int] = (1, 2, 3, 4, 5, 6), include_pos: Optional[str | Collection[str]] = ('NOUN', 'PROPN', 'ADJ'), window_size: int = 1500, topn: int | float = 10, idf: Optional[dict[str, float]] = None) -> list[tuple[str, float]]

Extract key terms from a document using the SGRank algorithm.

This comment is just taken straight from textacy, will adjust later

Parameters:

Name	Type	Description	Default
doc	Doc | str	spaCy Doc from which to extract keyterms.	required
normalize	Optional[str | Callable[[Span], str]]	If "lemma", lemmatize terms; if "lower", lowercase terms; if None, use the form of terms as they appeared in doc; if a callable, must accept a Span and return a str, e.g. :func:textacy.spacier.utils.get_normalized_text()	'lemma'
ngrams	int | Collection[int]	n of which n-grams to include. For example, (1, 2, 3, 4, 5, 6) (default) includes all ngrams from 1 to 6; 2 if only bigrams are wanted	(1, 2, 3, 4, 5, 6)
include_pos	Optional[str | Collection[str]]	One or more POS tags with which to filter for good candidate keyterms. If None, include tokens of all POS tags (which also allows keyterm extraction from docs without POS-tagging.)	('NOUN', 'PROPN', 'ADJ')
window_size	int	Size of sliding window in which term co-occurrences are determined to occur. Note: Larger values may dramatically increase runtime, owing to the larger number of co-occurrence combinations that must be counted.	1500
topn	int | float	Number of top-ranked terms to return as keyterms. If int, represents the absolute number; if float, must be in the open interval (0.0, 1.0), and is converted to an integer by int(round(len(candidates) * topn))	10
idf	Optional[dict[str, float]]	Mapping of normalize(term) to inverse document frequency for re-weighting of unigrams (n-grams with n > 1 have df assumed = 1). Results are typically better with idf information.	None

Returns:

Type	Description
list[tuple[str, float]]	Sorted list of top topn key terms and their corresponding SGRank scores

Raises:

Type	Description
ValueError	if topn is a float but not in (0.0, 1.0] or window_size < 2

References

Danesh, Sumner, and Martin. "SGRank: Combining Statistical and Graphical Methods to Improve the State of the Art in Unsupervised Keyphrase Extraction." Lexical and Computational Semantics (* SEM 2015) (2015): 117.

Source code in lexos/topwords/keyterms/sgrank.py

def sgrank(
    doc: Doc | str,
    *,
    normalize: Optional[str | Callable[[Span], str]] = "lemma",
    ngrams: int | Collection[int] = (1, 2, 3, 4, 5, 6),
    include_pos: Optional[str | Collection[str]] = ("NOUN", "PROPN", "ADJ"),
    window_size: int = 1500,
    topn: int | float = 10,
    idf: Optional[dict[str, float]] = None,
) -> list[tuple[str, float]]:
    """Extract key terms from a document using the SGRank algorithm.

    This comment is just taken straight from textacy, will adjust later

    Args:
        doc: spaCy `Doc` from which to extract keyterms.
        normalize: If "lemma", lemmatize terms; if "lower", lowercase terms; if None,
            use the form of terms as they appeared in `doc`; if a callable,
            must accept a `Span` and return a str,
            e.g. :func:`textacy.spacier.utils.get_normalized_text()`
        ngrams: n of which n-grams to include. For example, `(1, 2, 3, 4, 5, 6)` (default)
            includes all ngrams from 1 to 6; `2` if only bigrams are wanted
        include_pos: One or more POS tags with which to filter for good candidate keyterms.
            If None, include tokens of all POS tags
            (which also allows keyterm extraction from docs without POS-tagging.)
        window_size: Size of sliding window in which term co-occurrences are determined
            to occur. Note: Larger values may dramatically increase runtime, owing to
            the larger number of co-occurrence combinations that must be counted.
        topn: Number of top-ranked terms to return as keyterms.
            If int, represents the absolute number; if float, must be in the open interval
            (0.0, 1.0), and is converted to an integer by `int(round(len(candidates) * topn))`
        idf: Mapping of `normalize(term)` to inverse document frequency
            for re-weighting of unigrams (n-grams with n > 1 have df assumed = 1).
            Results are typically better with idf information.

    Returns:
        Sorted list of top `topn` key terms and their corresponding SGRank scores

    Raises:
        ValueError: if `topn` is a float but not in (0.0, 1.0] or `window_size` < 2

    References:
        Danesh, Sumner, and Martin. "SGRank: Combining Statistical and Graphical
        Methods to Improve the State of the Art in Unsupervised Keyphrase Extraction."
        Lexical and Computational Semantics (* SEM 2015) (2015): 117.
    """
    include_pos_set, ngram_sizes, topn = _validate_sgrank_args(
        include_pos, ngrams, window_size, topn
    )

    terms = _to_term_sequence(doc)
    if not terms:
        return []
    normalized_terms = list(terms_to_strings(terms, normalize))

    # Gather every valid n-gram candidate and its position in the doc.
    candidates = _get_candidates(terms, normalized_terms, include_pos_set, ngram_sizes)
    if not candidates:
        return []

    # Score each candidate a modified measure.
    term_weights = _compute_term_weights(candidates, idf)

    # Build a position-weighted co-occurrence graph and rank candidates within it.
    graph = _build_weighted_graph(candidates, term_weights, window_size)
    if graph.number_of_nodes() == 0:
        return []
    word_scores = nx.pagerank(graph, alpha=0.85, weight="weight")

    # Score thefull candidate phrases and limit overlapping candidates.
    candidate_scores = _score_candidate_phrases(candidates, word_scores)
    topn = _resolve_topn(topn, len(candidate_scores))
    sorted_candidate_scores = sorted(
        candidate_scores.items(), key=itemgetter(1, 0), reverse=True
    )

    return get_filtered_topn_terms(sorted_candidate_scores, topn, match_threshold=0.8)