3v324v23 commited on
Commit
a3db8d1
2 Parent(s): e2a62ec abd11e5

Merge branch 'v3.1' of github.com:binary-husky/chatgpt_academic into v3.1

Browse files
README.md CHANGED
@@ -1,4 +1,6 @@
1
- # ChatGPT 学术优化
 
 
2
 
3
  **如果喜欢这个项目,请给它一个Star;如果你发明了更好用的快捷键或函数插件,欢迎发issue或者pull requests**
4
 
@@ -68,11 +70,13 @@ huggingface免科学上网[在线体验](https://huggingface.co/spaces/qingxu98/
68
  <img src="https://user-images.githubusercontent.com/96192199/226935232-6b6a73ce-8900-4aee-93f9-733c7e6fef53.png" width="700" >
69
  </div>
70
 
71
- - 多种大语言模型混合调用([v3.1分支](https://github.com/binary-husky/chatgpt_academic/tree/v3.1)测试中)
72
  <div align="center">
73
- <img src="https://user-images.githubusercontent.com/96192199/231222778-34776885-a7f0-4f2c-b5f4-7cc2ef3ecb58.png" width="700" >
74
  </div>
75
 
 
 
76
 
77
 
78
  ## 直接运行 (Windows, Linux or MacOS)
 
1
+
2
+
3
+ # <img src="docs/logo.png" width="40" > ChatGPT 学术优化
4
 
5
  **如果喜欢这个项目,请给它一个Star;如果你发明了更好用的快捷键或函数插件,欢迎发issue或者pull requests**
6
 
 
70
  <img src="https://user-images.githubusercontent.com/96192199/226935232-6b6a73ce-8900-4aee-93f9-733c7e6fef53.png" width="700" >
71
  </div>
72
 
73
+ - 多种大语言模型混合调用(ChatGLM + OpenAI-GPT3.5 + [API2D](https://api2d.com/)-GPT4, [v3.1分支](https://github.com/binary-husky/chatgpt_academic/tree/v3.1)测试中)
74
  <div align="center">
75
+ <img src="https://user-images.githubusercontent.com/96192199/232537274-deca0563-7aa6-4b5d-94a2-b7c453c47794.png" width="700" >
76
  </div>
77
 
78
+ v3.1的[huggingface测试版](https://huggingface.co/spaces/qingxu98/academic-chatgpt-beta)(huggingface版不支持chatglm)
79
+
80
 
81
 
82
  ## 直接运行 (Windows, Linux or MacOS)
crazy_functions/crazy_utils.py CHANGED
@@ -448,6 +448,7 @@ def read_and_clean_pdf_text(fp):
448
  pf = 998
449
  for l in t['lines']:
450
  txt_line = "".join([wtf['text'] for wtf in l['spans']])
 
451
  pf = primary_ffsize(l)
452
  meta_line.append([txt_line, pf, l['bbox'], l])
453
  for wtf in l['spans']: # for l in t['lines']:
@@ -558,8 +559,8 @@ def read_and_clean_pdf_text(fp):
558
  meta_txt = meta_txt.replace('\n', '\n\n')
559
 
560
  ############################## <第 5 步,展示分割效果> ##################################
561
- for f in finals:
562
- print亮黄(f)
563
- print亮绿('***************************')
564
 
565
  return meta_txt, page_one_meta
 
448
  pf = 998
449
  for l in t['lines']:
450
  txt_line = "".join([wtf['text'] for wtf in l['spans']])
451
+ if len(txt_line) == 0: continue
452
  pf = primary_ffsize(l)
453
  meta_line.append([txt_line, pf, l['bbox'], l])
454
  for wtf in l['spans']: # for l in t['lines']:
 
559
  meta_txt = meta_txt.replace('\n', '\n\n')
560
 
561
  ############################## <第 5 步,展示分割效果> ##################################
562
+ # for f in finals:
563
+ # print亮黄(f)
564
+ # print亮绿('***************************')
565
 
566
  return meta_txt, page_one_meta
crazy_functions/批量翻译PDF文档_多线程.py CHANGED
@@ -13,7 +13,7 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_
13
  # 基本信息:功能、贡献者
14
  chatbot.append([
15
  "函数插件功能?",
16
- "批量总结PDF文档。函数插件贡献者: Binary-Husky"])
17
  yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
18
 
19
  # 尝试导入依赖,如果缺少依赖,则给出安装建议
@@ -59,7 +59,7 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_
59
  def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, sys_prompt):
60
  import os
61
  import tiktoken
62
- TOKEN_LIMIT_PER_FRAGMENT = 1600
63
  generated_conclusion_files = []
64
  for index, fp in enumerate(file_manifest):
65
 
@@ -91,13 +91,13 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot,
91
  # 多线,翻译
92
  gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
93
  inputs_array=[
94
- f"以下是你需要翻译的论文片段:\n{frag}" for frag in paper_fragments],
95
  inputs_show_user_array=[f"\n---\n 原文: \n\n {frag.replace('#', '')} \n---\n 翻译:\n " for frag in paper_fragments],
96
  llm_kwargs=llm_kwargs,
97
  chatbot=chatbot,
98
  history_array=[[paper_meta] for _ in paper_fragments],
99
  sys_prompt_array=[
100
- "请你作为一个学术翻译,负责把学术论文的片段准确翻译成中文。" for _ in paper_fragments],
101
  # max_workers=5 # OpenAI所允许的最大并行过载
102
  )
103
 
 
13
  # 基本信息:功能、贡献者
14
  chatbot.append([
15
  "函数插件功能?",
16
+ "批量翻译PDF文档。函数插件贡献者: Binary-Husky"])
17
  yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
18
 
19
  # 尝试导入依赖,如果缺少依赖,则给出安装建议
 
59
  def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, sys_prompt):
60
  import os
61
  import tiktoken
62
+ TOKEN_LIMIT_PER_FRAGMENT = 1280
63
  generated_conclusion_files = []
64
  for index, fp in enumerate(file_manifest):
65
 
 
91
  # 多线,翻译
92
  gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
93
  inputs_array=[
94
+ f"你需要翻译以下内容:\n{frag}" for frag in paper_fragments],
95
  inputs_show_user_array=[f"\n---\n 原文: \n\n {frag.replace('#', '')} \n---\n 翻译:\n " for frag in paper_fragments],
96
  llm_kwargs=llm_kwargs,
97
  chatbot=chatbot,
98
  history_array=[[paper_meta] for _ in paper_fragments],
99
  sys_prompt_array=[
100
+ "请你作为一个学术翻译,负责把学术论文准确翻译成中文。注意文章中的每一句话都要翻译。" for _ in paper_fragments],
101
  # max_workers=5 # OpenAI所允许的最大并行过载
102
  )
103
 
docs/Dockerfile+ChatGLM CHANGED
@@ -36,6 +36,9 @@ from transformers import AutoModel, AutoTokenizer \n\
36
  chatglm_tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) \n\
37
  chatglm_model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).float() ' >> warm_up_chatglm.py
38
  RUN python3 -u warm_up_chatglm.py
 
 
 
39
  RUN $useProxyNetwork git pull
40
 
41
  # 为chatgpt-academic配置代理和API-KEY (非必要 可选步骤)
 
36
  chatglm_tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) \n\
37
  chatglm_model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).float() ' >> warm_up_chatglm.py
38
  RUN python3 -u warm_up_chatglm.py
39
+
40
+ # 禁用缓存,确保更新代码
41
+ ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache
42
  RUN $useProxyNetwork git pull
43
 
44
  # 为chatgpt-academic配置代理和API-KEY (非必要 可选步骤)
docs/logo.png ADDED
main.py CHANGED
@@ -182,7 +182,7 @@ def main():
182
  threading.Thread(target=auto_update, name="self-upgrade", daemon=True).start()
183
 
184
  auto_opentab_delay()
185
- demo.queue(concurrency_count=CONCURRENT_COUNT).launch(server_name="0.0.0.0", server_port=PORT, auth=AUTHENTICATION)
186
 
187
  if __name__ == "__main__":
188
  main()
 
182
  threading.Thread(target=auto_update, name="self-upgrade", daemon=True).start()
183
 
184
  auto_opentab_delay()
185
+ demo.queue(concurrency_count=CONCURRENT_COUNT).launch(server_name="0.0.0.0", server_port=PORT, auth=AUTHENTICATION, favicon_path="docs/logo.png")
186
 
187
  if __name__ == "__main__":
188
  main()
request_llm/bridge_all.py CHANGED
@@ -23,6 +23,9 @@ from .bridge_tgui import predict as tgui_ui
23
 
24
  colors = ['#FF00FF', '#00FFFF', '#FF0000', '#990099', '#009999', '#990044']
25
 
 
 
 
26
  model_info = {
27
  # openai
28
  "gpt-3.5-turbo": {
@@ -31,7 +34,7 @@ model_info = {
31
  "endpoint": "https://api.openai.com/v1/chat/completions",
32
  "max_token": 4096,
33
  "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
34
- "token_cnt": lambda txt: len(tiktoken.encoding_for_model("gpt-3.5-turbo").encode(txt, disallowed_special=())),
35
  },
36
 
37
  "gpt-4": {
@@ -40,7 +43,7 @@ model_info = {
40
  "endpoint": "https://api.openai.com/v1/chat/completions",
41
  "max_token": 8192,
42
  "tokenizer": tiktoken.encoding_for_model("gpt-4"),
43
- "token_cnt": lambda txt: len(tiktoken.encoding_for_model("gpt-4").encode(txt, disallowed_special=())),
44
  },
45
 
46
  # api_2d
@@ -50,7 +53,7 @@ model_info = {
50
  "endpoint": "https://openai.api2d.net/v1/chat/completions",
51
  "max_token": 4096,
52
  "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
53
- "token_cnt": lambda txt: len(tiktoken.encoding_for_model("gpt-3.5-turbo").encode(txt, disallowed_special=())),
54
  },
55
 
56
  "api2d-gpt-4": {
@@ -59,7 +62,7 @@ model_info = {
59
  "endpoint": "https://openai.api2d.net/v1/chat/completions",
60
  "max_token": 8192,
61
  "tokenizer": tiktoken.encoding_for_model("gpt-4"),
62
- "token_cnt": lambda txt: len(tiktoken.encoding_for_model("gpt-4").encode(txt, disallowed_special=())),
63
  },
64
 
65
  # chatglm
@@ -69,7 +72,7 @@ model_info = {
69
  "endpoint": None,
70
  "max_token": 1024,
71
  "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
72
- "token_cnt": lambda txt: len(tiktoken.encoding_for_model("gpt-3.5-turbo").encode(txt, disallowed_special=())),
73
  },
74
 
75
  }
 
23
 
24
  colors = ['#FF00FF', '#00FFFF', '#FF0000', '#990099', '#009999', '#990044']
25
 
26
+ get_token_num_gpt35 = lambda txt: len(tiktoken.encoding_for_model("gpt-3.5-turbo").encode(txt, disallowed_special=()))
27
+ get_token_num_gpt4 = lambda txt: len(tiktoken.encoding_for_model("gpt-4").encode(txt, disallowed_special=()))
28
+
29
  model_info = {
30
  # openai
31
  "gpt-3.5-turbo": {
 
34
  "endpoint": "https://api.openai.com/v1/chat/completions",
35
  "max_token": 4096,
36
  "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
37
+ "token_cnt": get_token_num_gpt35,
38
  },
39
 
40
  "gpt-4": {
 
43
  "endpoint": "https://api.openai.com/v1/chat/completions",
44
  "max_token": 8192,
45
  "tokenizer": tiktoken.encoding_for_model("gpt-4"),
46
+ "token_cnt": get_token_num_gpt4,
47
  },
48
 
49
  # api_2d
 
53
  "endpoint": "https://openai.api2d.net/v1/chat/completions",
54
  "max_token": 4096,
55
  "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
56
+ "token_cnt": get_token_num_gpt35,
57
  },
58
 
59
  "api2d-gpt-4": {
 
62
  "endpoint": "https://openai.api2d.net/v1/chat/completions",
63
  "max_token": 8192,
64
  "tokenizer": tiktoken.encoding_for_model("gpt-4"),
65
+ "token_cnt": get_token_num_gpt4,
66
  },
67
 
68
  # chatglm
 
72
  "endpoint": None,
73
  "max_token": 1024,
74
  "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
75
+ "token_cnt": get_token_num_gpt35,
76
  },
77
 
78
  }
toolbox.py CHANGED
@@ -401,7 +401,7 @@ def on_file_uploaded(files, chatbot, txt, txt2, checkboxes):
401
  chatbot.append(['我上传了文件,请查收',
402
  f'[Local Message] 收到以下文件: \n\n{moved_files_str}' +
403
  f'\n\n调用路径参数已自动修正到: \n\n{txt}' +
404
- f'\n\n现在您点击任意实验功能时,以上文件将被作为输入参数'+err_msg])
405
  return chatbot, txt, txt2
406
 
407
 
 
401
  chatbot.append(['我上传了文件,请查收',
402
  f'[Local Message] 收到以下文件: \n\n{moved_files_str}' +
403
  f'\n\n调用路径参数已自动修正到: \n\n{txt}' +
404
+ f'\n\n现在您点击任意“红颜色”标识的函数插件时,以上文件将被作为输入参数'+err_msg])
405
  return chatbot, txt, txt2
406
 
407