First HF_SPace APp

.env

@@ -0,0 +1,10 @@
+# Directories
+IMAGES_DIR = images/
+OUTPUTS_DIR = outputs/
+PDF_DIR = pdfs/
+
+# Bucket
+MINIO_KEY = QUK5tI3fsjStPYrCKs18eb3OPTFzPLGeVOLXrsMc
+HOST = localhost:9000
+BUCKET_NAME = table-detection
+ACCESS_KEY = Table extraction

.gitignore

@@ -0,0 +1,3 @@
+junks/
+demo.py
+__pycache__

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.9-slim-bullseye
+
+WORKDIR /
+
+COPY requirements.txt requirements.txt
+
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+
+COPY . .
+EXPOSE $PORT
+
+CMD ["uvicorn", "main:app", "--reload"]     

README.md

@@ -1,10 +1,9 @@
----
-title: Table Detection
-emoji: 🏢
-colorFrom: purple
-colorTo: purple
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Table Detection
+
+- To run fast api server ```uvicorn main:app --reload```
+
+![FastAPI-Server](static/fastapi.png)
+
+- RUN MINIO SERVER:  .\minio.exe server C:\minio --console-address :9090
+
+![minIO-server](static/minIO.png)

folder_creation.py

@@ -0,0 +1,12 @@
+import os
+
+def remove_files(dirs):
+    for files in os.listdir(dirs):
+        os.remove(os.path.join(dirs, files))
+
+
+def make_directory_if_not_exists(dir_name):
+    if not os.path.exists(dir_name):
+        os.mkdir(dir_name)
+    else:
+        remove_files(dir_name)

main.py

@@ -0,0 +1,65 @@
+import os
+import shutil
+from pathlib import Path, PurePath
+
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi import FastAPI, Form, HTTPException
+from dotenv import load_dotenv
+
+from folder_creation import make_directory_if_not_exists
+from pdf_to_img import pdf_to_image
+from table_extraction import Table_extraction
+from minio_services import MINIO
+
+load_dotenv()
+
+IMAGES_DIR = os.getenv('IMAGES_DIR')
+PDF_DIR = os.getenv('PDF_DIR')
+OUTPUTS_DIR = os.getenv('OUTPUTS_DIR')
+HOST = os.getenv('HOST')
+ACCESS_KEY = os.getenv('ACCESS_KEY')
+MINIO_KEY = os.getenv('MINIO_KEY')
+BUCKET_NAME = os.getenv('BUCKET_NAME')
+app = FastAPI()
+
+origins = ["*"]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+@app.post("/")
+async def main(file_name: str = Form(None), uid: str = Form(None)):
+    try:
+        make_directory_if_not_exists(IMAGES_DIR)
+        IMAGES_DIR1 = IMAGES_DIR + uid + '/'
+        make_directory_if_not_exists(IMAGES_DIR1)
+
+        make_directory_if_not_exists(OUTPUTS_DIR)
+        OUTPUTS_DIR1 = OUTPUTS_DIR + uid + '/'
+        make_directory_if_not_exists(OUTPUTS_DIR1)
+        print(file_name)
+        if PurePath(file_name).suffix == '.pdf':
+            imagename = pdf_to_image(file_name, IMAGES_DIR1)
+        else:
+            imagename = IMAGES_DIR1+Path(file_name).name
+            shutil.copy(file_name, imagename)
+            
+        model = Table_extraction(imagename, OUTPUTS_DIR1)
+        op_img = model.get_results()
+
+        minio = MINIO(HOST, ACCESS_KEY, MINIO_KEY, BUCKET_NAME, uid, op_img)
+        minio.upload_to_minio()
+        obj = minio.download_from_minio()
+        shutil.rmtree(IMAGES_DIR)
+        shutil.rmtree(OUTPUTS_DIR)
+        return obj
+
+    except Exception as e:
+        shutil.rmtree(IMAGES_DIR)
+        shutil.rmtree(OUTPUTS_DIR)
+        raise HTTPException(status_code=404, detail=str(e))

minio_services.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+from minio import Minio
+from minio.error import (ResponseError, BucketAlreadyOwnedByYou,
+                         BucketAlreadyExists)
+class MINIO():
+    def __init__(self, HOST, ACCESS_KEY, MINIO_KEY, BUCKET_NAME, UID, op):
+        self.minioClient = Minio(HOST,
+                        access_key=ACCESS_KEY,
+                        secret_key=MINIO_KEY,
+                        secure=False)
+        self.BUCKET_NAME = BUCKET_NAME
+        self.UID = UID
+        self.op = op
+    
+    def upload_to_minio(self):
+        try:
+            self.minioClient.make_bucket(self.BUCKET_NAME, location="us-east-1")
+        except BucketAlreadyOwnedByYou as err:
+            pass
+        except BucketAlreadyExists as err:
+            pass
+        except ResponseError as err:
+            raise
+
+        # Put an object 'A' with contents from 'B'.
+        try:
+            self.minioClient.fput_object(self.BUCKET_NAME, str(self.UID) + '/' + Path(self.op).name, self.op)
+        except ResponseError as err:
+            print(err)
+    
+
+    def download_from_minio(self):
+        val = self.minioClient.fget_object(self.BUCKET_NAME, Path(self.op).name, str(self.UID) + '/' + Path(self.op).name)
+        return val.object_name
+
+

pdf_to_img.py

@@ -0,0 +1,26 @@
+import pypdfium2 as pdfium
+
+
+def pdf_to_image(pdf_name, IMAGES_DIR):
+    pdf = pdfium.PdfDocument(pdf_name)
+    n_pages = len(pdf)
+
+    for page_number in range(n_pages):
+        page = pdf.get_page(page_number)
+
+        scale_value = 3
+        pil_image = page.render_to(
+            pdfium.BitmapConv.pil_image,
+            scale = scale_value,
+            rotation = 0,
+            fill_colour=(255, 255, 255, 255),
+            crop=(0, 0, 0, 0),
+            greyscale=False,
+            optimise_mode=pdfium.OptimiseMode.NONE,)
+
+        imagename = IMAGES_DIR + str(page_number + 1) + ".png"
+        print(imagename)
+        pil_image.save(imagename)
+        return imagename
+        
+# pdf_to_image('junks\\Attention is all u need.pdf')

requirements.txt

@@ -0,0 +1,6 @@
+ultralyticsplus==0.0.23
+ultralytics==8.0.21
+transformers
+pypdfium2==3.15.0
+ak-minio==5.0.7.post2
+fastapi[all]

table_extraction.py

@@ -0,0 +1,65 @@
+from ultralyticsplus import YOLO, render_result
+from pathlib import Path, PurePath
+
+class Table_extraction():
+    
+    def __init__(self, image, OUTPUTS_DIR):
+        self.model = YOLO('keremberke/yolov8m-table-extraction')
+        self.model.overrides['conf'] = 0.25
+        self.model.overrides['iou'] = 0.45
+        self.model.overrides['agnostic_nms'] = False 
+        self.model.overrides['max_det'] = 1000
+        self.image = image
+        self.OUTPUTS_DIR = OUTPUTS_DIR
+
+    
+    def get_results(self):
+        self.results = self.model(self.image)
+        render = render_result(model=self.model, image=self.image, result=self.results[0])
+        op_img = self.OUTPUTS_DIR + Path(self.image).name
+        render.save(op_img)
+        return op_img
+
+    # def recognize_coords(self):
+    #     final_results = []
+    #     result = str(self.results)
+    #     print(result)
+    #     result = result.split('(')
+    #     print('Result', result)
+    #     if '\n' not in result[1]:
+    #         print('In if')
+    #         coords = []
+    #         result = result[1][2: -3].split(',')
+    #         for i in result:
+    #             coords.append(float(i))
+    #         final_results.append(coords)
+    #         return final_results
+    #     else:
+    #         result = result[1:][0]
+    #         result = result.split('\n')
+    #         j = 0
+    #         print('Results: ', result)
+    #         for i in result:
+    #             coords = []
+    #             i = i.strip()
+    #             if j == 0:
+    #                 print(i)
+    #                 i = i[2:-2]
+    #                 print(i)
+    #             elif j == len(result) - 1:
+    #                 i = i[1:-3]
+    #             else:
+    #                 i = i[1:-2]
+    #             j+=1
+    #             print(i)
+    #             i = i.split(',')
+    #             print(i)
+    #             for k in i:
+    #                 coords.append(float(k))
+    #             final_results.append(coords)
+    #     return final_results
+    
+
+# te = Table_extraction('junks\9.png', 'outputs/123')
+# te.get_results()
+# print(te.recognize_coords())