feat: add new filters

extract_config.yaml

@@ -4,7 +4,8 @@ pagefilter:
       modelfile: random_forest_model_low_false_positive.joblib
 
 table_extraction:
+  - type: LlamaParse
   - type: Unstructured
     params:
       hi_res_model_name: "yolox"
-      pdf_image_dpi: 300
+      pdf_image_dpi: 300

pages/0_Import_File.py

@@ -7,7 +7,7 @@ import yaml
 import copy
 from menu import display_pages_menu, display_config
 from pypdf import PdfReader
-from utils import get_pdf_iframe, set_state
+from utils import get_pdf_iframe, set_state, generate_assets
 
 from country_by_country.processor import ReportProcessor
 
@@ -18,6 +18,48 @@ def set_page_filter(value: dict):
     set_state(["config", "pagefilter"], value)
 
 
+def initiate_configuration() -> None:
+    st.session_state["config"] = copy.deepcopy(st.session_state["initial_config"])
+    if isinstance(st.session_state["config"]["pagefilter"], list):
+        st.session_state["config"]["pagefilter"] = st.session_state["initial_config"][
+            "pagefilter"
+        ][0]
+    st.session_state["selected_page_filter_name"] = st.session_state["config"][
+        "pagefilter"
+    ]["type"]
+
+
+def on_pdf_file_upload() -> None:
+    # Change states related to the pdf file upload
+    mytmpfile.write(st.session_state.original_pdf.read())
+    st.session_state["working_file_pdf"] = mytmpfile
+    st.session_state["original_pdf_name"] = st.session_state.original_pdf.name
+
+    # Generate assets
+    generate_assets()
+
+    st.session_state["page_redirection"] = "pages/1_Selected_Pages.py"
+
+
+def on_config_file_upload() -> None:
+    st.session_state["initial_config"] = st.session_state["initial_uploaded_config"]
+    initiate_configuration()
+
+
+def on_change_page_filter(name_to_filter_dict: dict) -> None:
+    st.session_state["selected_page_filter_name"] = st.session_state[
+        "radio_button_filter_selection"
+    ]  # this 'buffer' is needed because selectors wipe their key on reload
+    set_page_filter(name_to_filter_dict[st.session_state["selected_page_filter_name"]])
+
+
+# Check if a redirection was requested
+# Workaround because st.switch_page is not allowed in a callback function
+if st.session_state.get("page_redirection", False):
+    page_to_redirect_to = st.session_state["page_redirection"]
+    st.session_state["page_redirection"] = False
+    st.switch_page(page_to_redirect_to)
+
 st.set_page_config(layout="wide", page_title="Accueil - upload de PDF")
 st.title("Country by Country Tax Reporting analysis")
 st.subheader(
@@ -27,6 +69,23 @@ display_pages_menu()
 
 mytmpfile = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
 
+# State initialization
+if "first_time" not in st.session_state:
+    logging.info("State initialization...")
+    st.session_state["first_time"] = False
+
+    logging.info("... loading default extract config")
+    with open("extract_config.yaml", "r") as f:
+        st.session_state["initial_config"] = yaml.safe_load(f.read())
+    initiate_configuration()
+
+    logging.info("... initializing processor and assets")
+    st.session_state["proc"] = ReportProcessor(st.session_state["config"])
+    st.session_state["assets"] = {
+        "pagefilter": {},
+        "table_extractors": [],
+    }
+
 with st.sidebar:
 
     st.markdown("# PDF Upload")
@@ -34,13 +93,10 @@ with st.sidebar:
     st.markdown("## PDF Report to process")
     original_pdf = st.file_uploader(
         "Upload a pdf document containing financial table : ",
+        key="original_pdf",
+        on_change=on_pdf_file_upload,
     )
 
-    if original_pdf is not None:
-        mytmpfile.write(original_pdf.read())
-        st.session_state["working_file_pdf"] = mytmpfile
-        st.session_state["original_pdf_name"] = original_pdf.name
-
     if "original_pdf_name" in st.session_state:
         st.markdown(
             "Already loaded file : " + st.session_state["original_pdf_name"],
@@ -50,7 +106,10 @@ with st.sidebar:
     # Upload personalized config if required
     loaded_config = st.file_uploader(
         "Upload a config if the default config doesn't suit you :",
+        key="initial_uploaded_config",
+        on_change=initiate_configuration,
     )
+
     if loaded_config is not None:
         if not loaded_config.name.endswith(".yaml"):
             st.error("Please upload a yaml file")
@@ -69,26 +128,28 @@ with st.sidebar:
             loaded_config = None
 
     # Extract config
-    with open("extract_config.yaml", "r") as f:
-        default_config = f.read()
-
-    if not st.session_state.get("config_is_set", False):
-        st.session_state["initial_config"] = yaml.safe_load(default_config)
-        st.session_state["config"] = copy.deepcopy(st.session_state["initial_config"])
-        st.session_state["config_is_set"] = True
 
     if bool(loaded_config):
         st.session_state["initial_config"] = loaded_config_dict
         st.session_state["config"] = copy.deepcopy(st.session_state["initial_config"])
-        st.session_state["config_is_set"] = True
 
     # Set page filter
-    page_filter_radio_dict = {
+    page_filter_name_to_config_mapping = {
         pagefilter["type"]: pagefilter
         for pagefilter in st.session_state["initial_config"]["pagefilter"]
     }
-    selected_page_filter = st.radio("Page filter", page_filter_radio_dict.keys())
-    set_page_filter(page_filter_radio_dict[selected_page_filter])
+    page_filter_list = list(page_filter_name_to_config_mapping.keys())
+    current_selected_page_filter_index = page_filter_list.index(
+        st.session_state["selected_page_filter_name"]
+    )
+    selected_page_filter_name = st.radio(
+        "Page filter",
+        page_filter_list,
+        index=current_selected_page_filter_index,
+        on_change=on_change_page_filter,
+        key="radio_button_filter_selection",
+        args=(page_filter_name_to_config_mapping,),
+    )
 
     display_config()
 
@@ -103,32 +164,3 @@ if "working_file_pdf" in st.session_state:
         get_pdf_iframe(st.session_state["working_file_pdf"].name),
         unsafe_allow_html=True,
     )
-
-    if "first_time" not in st.session_state:
-        st.session_state["first_time"] = False
-        logging.info("Loading config and pdf")
-        st.session_state["proc"] = ReportProcessor(st.session_state["config"])
-
-        logging.info("Config and pdf loaded")
-
-        assets = {
-            "pagefilter": {},
-            "table_extractors": [],
-        }
-
-        # Filtering the pages
-        st.session_state["proc"].page_filter(
-            st.session_state["working_file_pdf"].name,
-            assets,
-        )
-
-        logging.info(f"Assets : {assets}")
-
-        if len(assets["pagefilter"]["selected_pages"]) == 0:
-            # No page has been automatically selected by the page filter
-            # Hence, we display the full pdf, letting the user select the pages
-            pdfreader = PdfReader(st.session_state["working_file_pdf"])
-            number_pages = len(PdfReader(st.session_state["working_file_pdf"]).pages)
-            assets["pagefilter"]["selected_pages"] = list(range(number_pages))
-        st.session_state["assets"] = assets
-        st.switch_page("pages/1_Selected_Pages.py")

pages/1_Selected_Pages.py

@@ -1,6 +1,6 @@
 import streamlit as st
 from country_by_country.processor import ReportProcessor
-from utils import get_pdf_iframe, set_state
+from utils import get_pdf_iframe, set_state, generate_assets
 from country_by_country.utils.utils import keep_pages
 from pypdf import PdfReader
 from menu import display_pages_menu, display_config
@@ -29,6 +29,7 @@ def set_extractors() -> None:
     ]
     set_state(["config", "table_extraction"], selected_extractors_dict)
     st.session_state["proc"] = ReportProcessor(st.session_state["config"])
+    generate_assets()
 
 
 st.set_page_config(layout="wide", page_title="Pages selection")  # page_icon="📈"

pages/2_Metadata.py

@@ -41,6 +41,7 @@ if "pdf_after_page_validation" in st.session_state:
                 currency = st.session_state["metadata"]["currency"]
                 unit = st.session_state["metadata"]["unit"]
                 headquarter = st.session_state["metadata"]["headquarter"]
+                decimal_separator = st.session_state["metadata"]["separator"]
             else:
                 company_name = None
                 sector = None
@@ -48,6 +49,15 @@ if "pdf_after_page_validation" in st.session_state:
                 currency = None
                 unit = None
                 headquarter = ""
+                decimal_separator = ","
+
+            separator_list = [",", "."]
+            decimal_separator = st.selectbox(
+                "Decimal separator",
+                separator_list,
+                index=separator_list.index(decimal_separator),
+            )
+
             companies = list(COMPANIES.keys())
             company_name = st.selectbox(
                 "Company name",
@@ -73,7 +83,9 @@ if "pdf_after_page_validation" in st.session_state:
             currency = st.selectbox(
                 "Currency",
                 currencies,
-                index=currencies.index(currency) if currency else currencies.index("EUR - Euro"),
+                index=currencies.index(currency)
+                if currency
+                else currencies.index("EUR - Euro"),
             )
 
             units = [
@@ -98,6 +110,7 @@ if "pdf_after_page_validation" in st.session_state:
             )
             if submitted:
                 st.session_state["metadata"] = {
+                    "separator": decimal_separator,
                     "company_name": company_name,
                     "sector": sector,
                     "year": year,

pages/5_Clean_Tables.py

@@ -50,12 +50,12 @@ def convert_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame:
     for column_name in dataframe.columns:
         try:
             dataframe[column_name] = dataframe[column_name].astype(float)
-        except Exception:
+        except Exception as e:
             pass
     return dataframe
 
 
-special_characters = "#&()[]@©"
+special_characters = "#&()[]@©€$'R¹³²"
 
 
 def style_symbol(v, props=""):
@@ -181,25 +181,52 @@ if (
             height=900,
         )
 
+    st.subheader(
+        "Filters : ",
+    )
+
     col7, col8, col9 = st.columns([1, 1, 1])
     with col7:
         total = st.checkbox(
             "Calculate the Total of each columns, excluding the last row", value=True
         )
         country = st.checkbox("Activate the country filter", value=True)
+        decimal_cleanup = st.checkbox("Apply decimal cleanup")
 
     with col8:
         negativ = st.checkbox(
             "Show the negative numbers, for each columns detected as a numerical type"
         )
+
+        with st.container(border=True):
+            cleanup_rules = st.checkbox(
+                "Apply clean up rules : (number) mean a negative number, o-> 0, homogenization NA, ect ect "
+            )
+            if cleanup_rules:
+                cleanup_excluded = st.multiselect(
+                    "exclude from filtering",
+                    st.session_state.tables[st.session_state["algorithm_name"]].columns,
+                    key="cleanup",
+                )
+
     with col9:
-        symbol = st.checkbox(
-            "Show the cells that contain a special symbol : " + special_characters,
-            value=True,
-        )
-        remove_symbols = st.checkbox("Remove the special symbols")
+        with st.container(border=True):
+            symbol = st.checkbox(
+                "Show the cells that contain a special symbol : " + special_characters,
+                value=True,
+            )
+            remove_symbols = st.checkbox(
+                "Remove the special symbols on numeric columns"
+            )
+            if remove_symbols:
+                rm_symbol_excluded = st.multiselect(
+                    "exclude from filtering",
+                    st.session_state.tables[st.session_state["algorithm_name"]].columns,
+                    key="rm_symbol",
+                )
 
     dataframe = st.session_state.tables[st.session_state["algorithm_name"]].copy()
+    dataframe = convert_dataframe(dataframe)
 
     if country:
         dataframe.iloc[:-2, 0] = dataframe.iloc[:-2, 0].apply(
@@ -207,13 +234,74 @@ if (
         )
 
     if remove_symbols:
-        pattern = "\(.*?\)" + "|[" + re.escape(special_characters) + "]"
-        for column in dataframe.columns:
-            dataframe[column] = dataframe[column].apply(
-                lambda x: re.sub(pattern, "", str(x))
-            )
+        pattern = "[" + re.escape(special_characters) + "]"
+        for column, dtype in dataframe.dtypes.items():
+            if column not in rm_symbol_excluded:
+                dataframe[column] = dataframe[column].apply(
+                    lambda x: re.sub(pattern, "", str(x))
+                )
         dataframe = convert_dataframe(dataframe)
 
+    if cleanup_rules:
+        for column, dtype in dataframe.dtypes.items():
+            if column not in cleanup_excluded:
+                # this is a code translated by chatgpt from Kane's R code
+                dataframe[column] = dataframe[column].replace(
+                    {"^-$|^$|^ $|^N/I$|^- -$|^N/A$|^n\\.a\\.$": None}, regex=True
+                )
+                dataframe[column] = dataframe[column].replace(
+                    {"^o$|^O$|^\\(o\\)$|^\\(O\\)$|^\\(0\\)$": "0"}, regex=True
+                )
+
+                if dtype == object:
+                    dataframe[column] = dataframe[column].str.replace(
+                        "(\\(.*\\))[:alnum:]+", "\\1", regex=True
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        "\\([:alnum:]+$|\\)[:alnum:]+$", "", regex=True
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        "\\([:alpha:]+\\)", "", regex=True
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        "(.+)\\(.+\\)$", "\\1", regex=True
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        "^\\(-(.*)\\)", "-\\1", regex=True
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        "^\\((.*)\\)", "-\\1", regex=True
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        "\\(.*\\)| |\\*|^-$|\\[.*\\]|^-€$", "", regex=True
+                    )
+        dataframe = convert_dataframe(dataframe)
+    if decimal_cleanup:
+        decimal_separator = (
+            st.session_state["metadata"]["separator"]
+            if st.session_state["metadata"]["separator"]
+            else ","
+        )
+        for column, dtype in dataframe.dtypes.items():
+            if dtype == object:
+                if decimal_separator == ",":
+                    dataframe[column] = dataframe[column].str.replace(
+                        "\\.", "", regex=False
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        ",", ".", regex=False
+                    )
+                else:
+                    dataframe[column] = dataframe[column].str.replace(
+                        ",(.{1,2})$", ".\\1", regex=True
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        "\\.([0-9]{3})", ",\\1", regex=True
+                    )
+                    dataframe[column] = dataframe[column].str.replace(
+                        ",", "", regex=False
+                    )
+
     if total:
         dataframe = convert_dataframe(dataframe)
         new_row = dataframe.apply(column_sum, axis=0)

utils.py

@@ -1,9 +1,11 @@
 import base64
+import logging
 from pathlib import Path
 from typing import Any
 
 import pandas as pd
 import streamlit as st
+from pypdf import PdfReader
 
 
 def get_pdf_iframe(pdf_to_process: str) -> str:
@@ -61,3 +63,25 @@ def set_state(key: Any, value: Any) -> None:
         nested_value[key_list[-1]] = value
     else:
         st.session_state[key] = value
+
+
+def generate_assets() -> None:
+    assets = {
+        "pagefilter": {},
+        "table_extractors": [],
+    }
+
+    # Filtering the pages
+    st.session_state["proc"].page_filter(
+        st.session_state["working_file_pdf"].name,
+        assets,
+    )
+
+    logging.info(f"Assets : {assets}")
+
+    if len(assets["pagefilter"]["selected_pages"]) == 0:
+        # No page has been automatically selected by the page filter
+        # Hence, we display the full pdf, letting the user select the pages
+        number_pages = len(PdfReader(st.session_state["working_file_pdf"]).pages)
+        assets["pagefilter"]["selected_pages"] = list(range(number_pages))
+    st.session_state["assets"] = assets