Spaces:
Runtime error
Runtime error
machineuser
commited on
Commit
•
a2706d6
1
Parent(s):
2ae8da8
Sync widgets demo
Browse files
packages/tasks/src/tokenizer-data.ts
CHANGED
@@ -21,7 +21,7 @@ export interface AddedToken {
|
|
21 |
single_word?: boolean;
|
22 |
}
|
23 |
export type SpecialTokensMap = {
|
24 |
-
[key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken;
|
25 |
};
|
26 |
/**
|
27 |
* Public interface for tokenizer config
|
|
|
21 |
single_word?: boolean;
|
22 |
}
|
23 |
export type SpecialTokensMap = {
|
24 |
+
[key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken | null;
|
25 |
};
|
26 |
/**
|
27 |
* Public interface for tokenizer config
|
packages/widgets/src/lib/components/InferenceWidget/widgets/ConversationalWidget/ConversationalWidget.svelte
CHANGED
@@ -200,7 +200,7 @@
|
|
200 |
const value = tokenizerConfig[key];
|
201 |
if (typeof value === "string") {
|
202 |
specialTokensMap[key] = value;
|
203 |
-
} else if (typeof value === "object" && value.__type === "AddedToken" && value.content) {
|
204 |
specialTokensMap[key] = (value as AddedToken).content;
|
205 |
}
|
206 |
}
|
|
|
200 |
const value = tokenizerConfig[key];
|
201 |
if (typeof value === "string") {
|
202 |
specialTokensMap[key] = value;
|
203 |
+
} else if (typeof value === "object" && !!value && value.__type === "AddedToken" && value.content) {
|
204 |
specialTokensMap[key] = (value as AddedToken).content;
|
205 |
}
|
206 |
}
|