compliance_checks/evaluation.py

from compliance_checks.base import ComplianceResult, ComplianceCheck, walk_to_next_heading
from bs4 import BeautifulSoup


class EvaluationResult(ComplianceResult):
    name = "Evaluation and Metrics"

    def __init__(
            self,
            *args,
            **kwargs,
    ):
        super().__init__(*args, **kwargs)

    def __eq__(self, other):
        if isinstance(other, EvaluationResult):
            if super().__eq__(other):
                try:
                    return True
                except AssertionError:
                    return False
        else:
            return False

    def to_string(self):
        if self.status:
            return """\
            It looks like this model card has some documentation for how the model was evaluated! We look for this by \
            searching for headings that say things like:
            - Evaluation
            - Evaluation results
            - Benchmarks
            - Results
            """
        else:
            return """\
            We weren't able to find a section in this model card that reports the evaluation process, but it's easy to \
            add one! You can add the following section to the model card and, once you fill in the \
            `[More Information Needed]` sections, the "Evaluation and Metrics" check should pass 🤗
            
            ```md
            ## Evaluation
            
            <!-- This section describes the evaluation protocols and provides the results. -->
            
            ### Testing Data, Factors & Metrics
            
            #### Testing Data
            
            <!-- This should link to a Data Card if possible. -->
            
            [More Information Needed]
            
            #### Factors
            
            <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
            
            [More Information Needed]
            
            #### Metrics
            
            <!-- These are the evaluation metrics being used, ideally with a description of why. -->
            
            [More Information Needed]
            
            ### Results
            
            [More Information Needed]
            
            #### Summary
            
            [More Information Needed]
            ```
            """


class EvaluationCheck(ComplianceCheck):
    name = "Evaluation and Metrics"

    def run_check(self, card: BeautifulSoup):
        combos = [
            ("h1", "Evaluation"), ("h2", "Evaluation"),
            ("h2", "Evaluation results"), ("h2", "Evaluation Results"),
            ("h2", "Benchmarks"),
            ("h2", "Results"),
            ("h1", "Evaluation data"),
            ("h2", "Performance"),
        ]

        for hX, heading in combos:
            purpose_check = walk_to_next_heading(card, hX, heading)
            if purpose_check:
                return EvaluationResult(
                    status=True,
                )

        return EvaluationResult()