machineuser commited on
Commit
a2706d6
1 Parent(s): 2ae8da8

Sync widgets demo

Browse files
packages/tasks/src/tokenizer-data.ts CHANGED
@@ -21,7 +21,7 @@ export interface AddedToken {
21
  single_word?: boolean;
22
  }
23
  export type SpecialTokensMap = {
24
- [key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken;
25
  };
26
  /**
27
  * Public interface for tokenizer config
 
21
  single_word?: boolean;
22
  }
23
  export type SpecialTokensMap = {
24
+ [key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken | null;
25
  };
26
  /**
27
  * Public interface for tokenizer config
packages/widgets/src/lib/components/InferenceWidget/widgets/ConversationalWidget/ConversationalWidget.svelte CHANGED
@@ -200,7 +200,7 @@
200
  const value = tokenizerConfig[key];
201
  if (typeof value === "string") {
202
  specialTokensMap[key] = value;
203
- } else if (typeof value === "object" && value.__type === "AddedToken" && value.content) {
204
  specialTokensMap[key] = (value as AddedToken).content;
205
  }
206
  }
 
200
  const value = tokenizerConfig[key];
201
  if (typeof value === "string") {
202
  specialTokensMap[key] = value;
203
+ } else if (typeof value === "object" && !!value && value.__type === "AddedToken" && value.content) {
204
  specialTokensMap[key] = (value as AddedToken).content;
205
  }
206
  }