chat-aww

Build error

coyotte508 HF staff commited on Apr 25, 2023

Commit

a11cd21

•

1 Parent(s): 97b65ef

🎨 Remove more stopwords (#102)

Files changed (1) hide show

src/lib/components/chat/ChatMessage.svelte CHANGED Viewed

@@ -9,8 +9,10 @@
 	function sanitizeMd(md: string) {
 		return md
-			.replaceAll(/^(<\|startoftext\|>|<\|startoftext\|$|<\|startoftext$|<\|$|<$)/g, "")
-			.replaceAll(/(<\|endoftext\|>|<\|endoftext\|$|<\|endoftext$)/g, "")
 			.replaceAll("&", "&amp;")
 			.replaceAll("<", "&lt;");
 	}

 	function sanitizeMd(md: string) {
 		return md
+			.replaceAll(/^(<\|startoftext\|$|<\|startoftext$|<\|$|<$)/g, "")
+			.replaceAll(/(<\|endoftext\|$|<\|endoftext$)/g, "")
+			.replaceAll(/<\|[a-z]+\|>/g, " ")
+			.trim()
 			.replaceAll("&", "&amp;")
 			.replaceAll("<", "&lt;");
 	}