Update app.py

app.py

@@ -101,7 +101,7 @@ def deidentify_doc(pdftext, maxtokens, temperature, top_probability):
     #             n_ctx=8192,
     #             )
 
-    chat_template = MessagesFormatterType.LLAMA_3
+    # chat_template = MessagesFormatterType.LLAMA_3
     
     llm = Llama(
         model_path="models/Meta-Llama-3-70B-Instruct-Q3_K_M.gguf",
@@ -111,51 +111,51 @@ def deidentify_doc(pdftext, maxtokens, temperature, top_probability):
         n_ctx=8192,
     )
 
-    provider = LlamaCppPythonProvider(llm)
+    # provider = LlamaCppPythonProvider(llm)
     
-    agent = LlamaCppAgent(
-        provider,
-        system_prompt="You are a helpful assistant.",
-        predefined_messages_formatter_type=chat_template,
-        debug_output=True
-    )
+    # agent = LlamaCppAgent(
+    #     provider,
+    #     system_prompt="You are a helpful assistant.",
+    #     predefined_messages_formatter_type=chat_template,
+    #     debug_output=True
+    # )
     
-    settings = provider.get_provider_default_settings()
-    settings.temperature = 0.7
-    settings.top_k = 40
-    settings.top_p = 0.95
-    settings.max_tokens = 2048
-    settings.repeat_penalty = 1.1
-    settings.stream = True
-
-    messages = BasicChatHistory()
-
-    stream = agent.get_chat_response(
-        prompt + ' : ' + pdftext,
-        llm_sampling_settings=settings,
-        chat_history=messages,
-        returns_streaming_generator=True,
-        print_output=False
-    )
+    # settings = provider.get_provider_default_settings()
+    # settings.temperature = 0.7
+    # settings.top_k = 40
+    # settings.top_p = 0.95
+    # settings.max_tokens = 2048
+    # settings.repeat_penalty = 1.1
+    # settings.stream = True
+
+    # messages = BasicChatHistory()
+
+    # stream = agent.get_chat_response(
+    #     prompt + ' : ' + pdftext,
+    #     llm_sampling_settings=settings,
+    #     chat_history=messages,
+    #     returns_streaming_generator=True,
+    #     print_output=False
+    # )
 
-    outputs = ""
-    for output in stream:
-        outputs += output
+    # outputs = ""
+    # for output in stream:
+    #     outputs += output
     
-    return outputs
+    # return outputs
 
-    # output = model.create_chat_completion(
-    #     messages=[
-    #         {"role": "assistant", "content": prompt},
-    #         {
-    #             "role": "user",
-    #             "content": pdftext
-    #         }
-    #     ],
-    #     max_tokens=maxtokens,
-    #     temperature=temperature
-    # )
-    # output = output['choices'][0]['message']['content']
+    output = model.create_chat_completion(
+        messages=[
+            {"role": "assistant", "content": prompt},
+            {
+                "role": "user",
+                "content": pdftext
+            }
+        ],
+        max_tokens=maxtokens,
+        temperature=temperature
+    )
+    output = output['choices'][0]['message']['content']
 
     # prompt = "Perform the following actions on given text: 1. Replace any person age with term [redacted] 2. DO NOT REPLACE ANY MEDICAL MEASUREMENTS 3. Replace only the CALENDAR DATES of format 'day/month/year' with term [redacted]"
     # output = model.create_chat_completion(
@@ -175,7 +175,7 @@ def deidentify_doc(pdftext, maxtokens, temperature, top_probability):
     # print(output)
     # print('-------------------------------------------------------')
 
-    # return outputs
+    return output
 
 def pdf_to_text(files, maxtokens=2048, temperature=0, top_probability=0.95):
     print('Control 0-----------------------------------')
@@ -204,7 +204,7 @@ output_text = gr.Textbox()
 output_path_component = gr.File(label="Select Output Path")
 iface = gr.Interface(
     fn=pdf_to_text,
-    inputs='file',
+    inputs=['file', max_tokens, temp_slider, prob_slider],
     # inputs=["textbox", input_folder_text, "textbox", max_tokens, temp_slider, prob_slider],
     outputs=output_text,
     title='COBIx Endoscopy Report De-Identification',