Add initial instructional copy

app.py

@@ -25,7 +25,7 @@ def run_compliance_check(model_card: str):
     results = suite.run(model_card)
 
     return [
-        *[gr.Accordion.update(label=f"{r.name} - {status_emoji(r.status)}") for r in results],
+        *[gr.Accordion.update(label=f"{r.name} - {status_emoji(r.status)}", open=not r.status) for r in results],
         *[gr.Markdown.update(value=r.to_string()) for r in results],
     ]
 
@@ -57,14 +57,26 @@ with gr.Blocks(css="""\
 #file-upload .boundedheight {
     max-height: 100px;
 }
+
+code {
+    overflow: scroll;
+}
 """) as demo:
     gr.Markdown("""\
     # RegCheck AI
-    This Space uses model cards’ information as a source of regulatory compliance with some provisions of the proposed \
+    This Space uses [model cards’](https://huggingface.co/docs/hub/model-cards) information as a source of regulatory \
+    compliance with some provisions of the proposed \
     [EU AI Act](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=celex%3A52021PC0206). For the moment being, the \
     demo is a **prototype** limited to specific provisions of Article 13 of the AI Act, related to “Transparency and \
-    provision of information to users”. Choose a model card and check whether it has some useful info to comply with \
-    the EU AI Act! **(DISCLAIMER: this is NOT a commercial or legal advice-related product)**\
+    provision of information to users”. **(DISCLAIMER: this is NOT a commercial or legal advice-related product)**
+    
+    To check a model card, first load it by doing any one of the following:
+    - If the model is on the Hugging Face Hub, enter its model ID and click "Load model card".
+    - If you have the model card on your computer as a Markdown file, select the "Upload your own card" tab and click \
+      "Upload a Markdown file".
+    - Paste your model card's text directly into the "Model Card" text area.
+    
+    Once your card is loaded, click "Run validation checks" to receive your results.
     """)
 
     with gr.Row(elem_id="reverse-row"):

compliance_checks/base.py

@@ -21,7 +21,12 @@ def walk_to_next_heading(card, heading, heading_text) -> bool:
                 content.append(sibling.text.strip())
             sibling = next(sibling_gen, None)
 
-        if all([c in ["[More Information Needed]", "More information needed."] for c in content]):
+        if all([c in [
+            "[More Information Needed]",
+            "More information needed.",
+            "Users (both direct and downstream) should be made aware of the risks, biases and limitations of the "
+            "model. More information needed for further recommendations."
+        ] for c in content]):
             return False  # , None
 
         return True  # , content

compliance_checks/computational_requirements.py

@@ -27,16 +27,50 @@ class ComputationalRequirementsResult(ComplianceResult):
             return False
 
     def to_string(self):
-        return self.requirements
+        if self.status:
+            return """\
+            In order for users to know what kind of hardware and software they need to run a model, a model card \
+            should have information about the model's computational requirements. We found some documentation \
+            for this in this model card. We look for this by searching for a heading called "Technical Specifications".
+            """
+        else:
+            return """\
+            We weren't able to find a section in this model card for the model's computational requirements, but it's \
+            easy to add one! You can add the following section to the model card and, once you fill in the \
+            `[More Information Needed]` sections, the "Computational Requirements" check should pass 🤗
+
+            ```md
+            ## Technical Specifications [optional]
+            
+            ### Compute Infrastructure
+            
+            [More Information Needed]
+            
+            #### Hardware
+            
+            [More Information Needed]
+            
+            #### Software
+            
+            [More Information Needed]
+            ```
+            """
 
 
 class ComputationalRequirementsCheck(ComplianceCheck):
     name = "Computational Requirements"
 
     def run_check(self, card: BeautifulSoup):
-        check = walk_to_next_heading(card, "h2", "Technical Specifications")
+        combos = [
+            ("h2", "Technical Specifications"),
+            ("h2", "Technical Specifications [optional]"),
+        ]
+
+        for hX, heading in combos:
+            purpose_check = walk_to_next_heading(card, hX, heading)
+            if purpose_check:
+                return ComputationalRequirementsResult(
+                    status=True,
+                )
 
-        return ComputationalRequirementsResult(
-            status=check,
-            # requirements=content,
-        )
+        return ComputationalRequirementsResult()

compliance_checks/general_limitations.py

@@ -26,7 +26,34 @@ class GeneralLimitationsResult(ComplianceResult):
             return False
 
     def to_string(self):
-        return self.limitations
+        if self.status:
+            return """\
+            It's important for model cards to document the model's general limitations! We found some documentation \
+            for this in this model card. We look for this by searching for headings that say things like:
+            - Bias, Risks, and Limitations
+            - Intended uses & limitations
+            - Limitations
+            """
+        else:
+            return """\
+            We weren't able to find a section in this model card for the model's limitations, but it's easy to \
+            add one! You can add the following section to the model card and, once you fill in the \
+            `[More Information Needed]` sections, the "General Limitations" check should pass 🤗
+
+            ```md
+            ## Bias, Risks, and Limitations
+            
+            <!-- This section is meant to convey both technical and sociotechnical limitations. -->
+            
+            [More Information Needed]
+            
+            ### Recommendations
+            
+            <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+            
+            Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+            ```
+            """
 
 
 class GeneralLimitationsCheck(ComplianceCheck):

compliance_checks/intended_purpose.py

@@ -33,7 +33,45 @@ class IntendedPurposeResult(ComplianceResult):
             return False
 
     def to_string(self):
-        return str((self.direct_use, self.direct_use, self.out_of_scope_use))
+        if self.status:
+            return """\
+            It looks like this model card has some documentation for the model's intended purpose! We look for this by \
+            searching for headings that say things like:
+            - Intended uses & limitations
+            - Uses
+            - Model Use
+            """
+        else:
+            return """\
+            We weren't able to find a section in this model card for the model's intended purpose, but it's easy to \
+            add one! You can add the following section to the model card and, once you fill in the \
+            `[More Information Needed]` sections, the "Intended Purpose" check should pass 🤗
+            
+            ```md
+            ## Uses
+            
+            <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+            [More Information Needed]
+            
+            ### Direct Use
+            
+            <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+            
+            [More Information Needed]
+            
+            ### Downstream Use [optional]
+            
+            <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+            
+            [More Information Needed]
+            
+            ### Out-of-Scope Use
+            
+            <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+            [More Information Needed]
+            ```
+            """
 
 
 class IntendedPurposeCheck(ComplianceCheck):

tests/test_computational_requirements_check.py

@@ -7,6 +7,21 @@ from compliance_checks import (
 )
 
 
+empty_template = """\
+## Technical Specifications [optional]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+"""
 model_card_template = """\
 # Model Card for Sample Model
 
@@ -41,3 +56,10 @@ def test_run_checks(card):
     results = ComputationalRequirementsCheck().run_check(card_soup)
 
     assert results == success_result
+
+
+def test_fail_on_empty_template():
+    model_card_html = markdown.markdown(empty_template)
+    card_soup = BeautifulSoup(model_card_html, features="html.parser")
+    results = ComputationalRequirementsCheck().run_check(card_soup)
+    assert results == ComputationalRequirementsResult()

tests/test_general_limitations_check.py

@@ -6,7 +6,19 @@ from compliance_checks import (
     GeneralLimitationsCheck, GeneralLimitationsResult,
 )
 
+empty_template = """\
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
 
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+"""
 model_card_template = """\
 # Model Card for Sample Model
 
@@ -99,3 +111,10 @@ def test_run_checks(card):
     results = GeneralLimitationsCheck().run_check(card_soup)
 
     assert results == success_result
+
+
+def test_fail_on_empty_template():
+    model_card_html = markdown.markdown(empty_template)
+    card_soup = BeautifulSoup(model_card_html, features="html.parser")
+    results = GeneralLimitationsCheck().run_check(card_soup)
+    assert results == GeneralLimitationsResult()

tests/test_intended_purpose_check.py

@@ -6,15 +6,17 @@ from compliance_checks.intended_purpose import (
     IntendedPurposeCheck, IntendedPurposeResult,
 )
 
-
-model_card_template = """\
+empty_template = """\
 ## Uses
 
 <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+[More Information Needed]
 
 ### Direct Use
 
-Here is some info about direct uses...
+[More Information Needed]
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
 
 ### Downstream Use [optional]
 
@@ -25,6 +27,22 @@ Here is some info about direct uses...
 ### Out-of-Scope Use
 
 <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+[More Information Needed]
+"""
+model_card_template = """\
+## Uses
+
+Some info...
+
+### Direct Use
+
+Some more info.
+
+### Downstream Use [optional]
+
+[More Information Needed]
+
+### Out-of-Scope Use
 
 Here is some info about out-of-scope uses...
 """
@@ -101,3 +119,10 @@ def test_run_checks(card):
     results = IntendedPurposeCheck().run_check(card_soup)
 
     assert results == success_result
+
+
+def test_fail_on_empty_template():
+    model_card_html = markdown.markdown(empty_template)
+    card_soup = BeautifulSoup(model_card_html, features="html.parser")
+    results = IntendedPurposeCheck().run_check(card_soup)
+    assert results == IntendedPurposeResult()