tokenizer_config.json 905 B

12345678910111213141516171819202122232425262728293031323334
  1. {
  2. "unk_token": {
  3. "content": "<|endoftext|>",
  4. "single_word": false,
  5. "lstrip": false,
  6. "rstrip": false,
  7. "normalized": true,
  8. "__type": "AddedToken"
  9. },
  10. "bos_token": {
  11. "content": "<|startoftext|>",
  12. "single_word": false,
  13. "lstrip": false,
  14. "rstrip": false,
  15. "normalized": true,
  16. "__type": "AddedToken"
  17. },
  18. "eos_token": {
  19. "content": "<|endoftext|>",
  20. "single_word": false,
  21. "lstrip": false,
  22. "rstrip": false,
  23. "normalized": true,
  24. "__type": "AddedToken"
  25. },
  26. "pad_token": "<|endoftext|>",
  27. "add_prefix_space": false,
  28. "errors": "replace",
  29. "do_lower_case": true,
  30. "name_or_path": "openai/clip-vit-base-patch32",
  31. "model_max_length": 77,
  32. "special_tokens_map_file": "./special_tokens_map.json",
  33. "tokenizer_class": "CLIPTokenizer"
  34. }