{"crate":{"id":"tokenizers","name":"tokenizers","updated_at":"2026-09-03T08:49:28.624173Z","versions":[3142113,2315437,1854803,1736885,1705867,1660713,1627380,1613727,1481317,1354675,1354651,1353675,1328781,1327317,1297016,1227941,1118139,1117627,1048482,1022543,954784,917767,891526,891416,871541,768694,658737,635649,624218,524877,507797,420113,229345,229036,226087,217458,209768,202831,181582,168405,168162],"keywords":["bpe","huggingface","nlp","tokenizer","wordpiece"],"categories":[],"badges":[],"created_at":"2019-08-08T14:55:49.017781Z","downloads":30597477,"recent_downloads":12175913,"default_version":"0.23.2","num_versions":41,"yanked":false,"max_version":"0.23.2","newest_version":"0.23.2","max_stable_version":"0.23.2","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","links":{"version_downloads":"/api/v1/crates/tokenizers/downloads","versions":null,"owners":"/api/v1/crates/tokenizers/owners","owner_team":"/api/v1/crates/tokenizers/owner_team","owner_user":"/api/v1/crates/tokenizers/owner_user","reverse_dependencies":"/api/v1/crates/tokenizers/reverse_dependencies"},"exact_match":false,"trustpub_only":false},"versions":[{"id":3142113,"crate":"tokenizers","num":"0.23.2","dl_path":"/api/v1/crates/tokenizers/0.23.2/download","readme_path":"/api/v1/crates/tokenizers/0.23.2/readme","updated_at":"2026-09-03T08:49:28.624173Z","created_at":"2026-09-03T08:49:28.624173Z","downloads":9454,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"parity-aware-bpe":[],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/wasm_js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.23.2/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.23.2/downloads","authors":"/api/v1/crates/tokenizers/0.23.2/authors"},"crate_size":215739,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2026-09-03T08:49:28.624173Z"}],"checksum":"7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":21468,"comment_lines":885,"files":73}},"total_code_lines":21484,"total_comment_lines":885}},{"id":2315437,"crate":"tokenizers","num":"0.23.1","dl_path":"/api/v1/crates/tokenizers/0.23.1/download","readme_path":"/api/v1/crates/tokenizers/0.23.1/readme","updated_at":"2026-04-27T14:39:17.479740Z","created_at":"2026-04-27T14:39:17.479740Z","downloads":1711031,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/wasm_js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.23.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.23.1/downloads","authors":"/api/v1/crates/tokenizers/0.23.1/authors"},"crate_size":196225,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2026-04-27T14:39:17.479740Z"}],"checksum":"44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19785,"comment_lines":700,"files":72}},"total_code_lines":19801,"total_comment_lines":700}},{"id":1854803,"crate":"tokenizers","num":"0.22.2","dl_path":"/api/v1/crates/tokenizers/0.22.2/download","readme_path":"/api/v1/crates/tokenizers/0.22.2/readme","updated_at":"2025-12-02T12:53:51.498043Z","created_at":"2025-12-02T12:53:51.498043Z","downloads":10112389,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/wasm_js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.22.2/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.22.2/downloads","authors":"/api/v1/crates/tokenizers/0.22.2/authors"},"crate_size":186345,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2025-12-02T12:53:51.498043Z"}],"checksum":"b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19261,"comment_lines":658,"files":72}},"total_code_lines":19277,"total_comment_lines":658}},{"id":1736885,"crate":"tokenizers","num":"0.22.1","dl_path":"/api/v1/crates/tokenizers/0.22.1/download","readme_path":"/api/v1/crates/tokenizers/0.22.1/readme","updated_at":"2025-09-19T09:46:10.340856Z","created_at":"2025-09-19T09:46:10.340856Z","downloads":1404039,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/wasm_js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.22.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.22.1/downloads","authors":"/api/v1/crates/tokenizers/0.22.1/authors"},"crate_size":186019,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2025-09-19T09:46:10.340856Z"}],"checksum":"6475a27088c98ea96d00b39a9ddfb63780d1ad4cceb6f48374349a96ab2b7842","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19260,"comment_lines":661,"files":72}},"total_code_lines":19276,"total_comment_lines":661}},{"id":1705867,"crate":"tokenizers","num":"0.22.0","dl_path":"/api/v1/crates/tokenizers/0.22.0/download","readme_path":"/api/v1/crates/tokenizers/0.22.0/readme","updated_at":"2025-08-29T10:22:26.656448Z","created_at":"2025-08-29T10:22:26.656448Z","downloads":88331,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/wasm_js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.22.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.22.0/downloads","authors":"/api/v1/crates/tokenizers/0.22.0/authors"},"crate_size":185810,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2025-08-29T10:22:26.656448Z"}],"checksum":"af10f51be57162b69d90a15cb226eef12c9e4faecbd5e3ea98a86bfb920b3d71","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19260,"comment_lines":661,"files":72}},"total_code_lines":19276,"total_comment_lines":661}},{"id":1660713,"crate":"tokenizers","num":"0.21.4","dl_path":"/api/v1/crates/tokenizers/0.21.4/download","readme_path":"/api/v1/crates/tokenizers/0.21.4/readme","updated_at":"2025-07-28T13:19:38.577556Z","created_at":"2025-07-28T13:19:38.577556Z","downloads":8342152,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/wasm_js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.21.4/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.21.4/downloads","authors":"/api/v1/crates/tokenizers/0.21.4/authors"},"crate_size":185676,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2025-07-28T13:19:38.577556Z"}],"checksum":"a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19232,"comment_lines":664,"files":72}},"total_code_lines":19248,"total_comment_lines":664}},{"id":1627380,"crate":"tokenizers","num":"0.21.4-dev.0","dl_path":"/api/v1/crates/tokenizers/0.21.4-dev.0/download","readme_path":"/api/v1/crates/tokenizers/0.21.4-dev.0/readme","updated_at":"2025-07-04T11:58:54.408296Z","created_at":"2025-07-04T11:58:54.408296Z","downloads":492,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/wasm_js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.21.4-dev.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.21.4-dev.0/downloads","authors":"/api/v1/crates/tokenizers/0.21.4-dev.0/authors"},"crate_size":185636,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2025-07-04T11:58:54.408296Z"}],"checksum":"2fffc211fe4fea22d07f80d132d8ee64627af859189fb3422e84e3e1cd0a28d0","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19232,"comment_lines":664,"files":72}},"total_code_lines":19248,"total_comment_lines":664}},{"id":1613727,"crate":"tokenizers","num":"0.21.2","dl_path":"/api/v1/crates/tokenizers/0.21.2/download","readme_path":"/api/v1/crates/tokenizers/0.21.2/readme","updated_at":"2025-06-24T10:21:08.580505Z","created_at":"2025-06-24T10:21:08.580505Z","downloads":1684643,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/wasm_js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.21.2/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.21.2/downloads","authors":"/api/v1/crates/tokenizers/0.21.2/authors"},"crate_size":184673,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2025-06-24T10:21:08.580505Z"}],"checksum":"4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19226,"comment_lines":664,"files":72}},"total_code_lines":19242,"total_comment_lines":664}},{"id":1481317,"crate":"tokenizers","num":"0.21.1","dl_path":"/api/v1/crates/tokenizers/0.21.1/download","readme_path":"/api/v1/crates/tokenizers/0.21.1/readme","updated_at":"2025-03-13T10:45:44.334185Z","created_at":"2025-03-13T10:45:44.334185Z","downloads":1124989,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"rustls-tls":["hf-hub?/rustls-tls"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.21.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.21.1/downloads","authors":"/api/v1/crates/tokenizers/0.21.1/authors"},"crate_size":183678,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2025-03-13T10:45:44.334185Z"}],"checksum":"3169b3195f925496c895caee7978a335d49218488ef22375267fba5a46a40bd7","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19054,"comment_lines":646,"files":71}},"total_code_lines":19070,"total_comment_lines":646}},{"id":1354675,"crate":"tokenizers","num":"0.21.0","dl_path":"/api/v1/crates/tokenizers/0.21.0/download","readme_path":"/api/v1/crates/tokenizers/0.21.0/readme","updated_at":"2024-11-27T13:06:48.784354Z","created_at":"2024-11-27T13:06:48.784354Z","downloads":756590,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.21.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.21.0/downloads","authors":"/api/v1/crates/tokenizers/0.21.0/authors"},"crate_size":180128,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-11-27T13:06:48.784354Z"}],"checksum":"9ecededfed68a69bc657e486510089e255e53c3d38cc7d4d59c8742668ca2cae","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19092,"comment_lines":646,"files":71}},"total_code_lines":19108,"total_comment_lines":646}},{"id":1354651,"crate":"tokenizers","num":"0.21.0-rc0","dl_path":"/api/v1/crates/tokenizers/0.21.0-rc0/download","readme_path":"/api/v1/crates/tokenizers/0.21.0-rc0/readme","updated_at":"2024-11-27T12:55:55.183177Z","created_at":"2024-11-27T12:55:55.183177Z","downloads":824,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.21.0-rc0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.21.0-rc0/downloads","authors":"/api/v1/crates/tokenizers/0.21.0-rc0/authors"},"crate_size":180178,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-11-27T12:55:55.183177Z"}],"checksum":"b335408745e6c2b69941f4ca1df2cfd70669f0ece096d72f0f06624d69abfc1b","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19092,"comment_lines":646,"files":71}},"total_code_lines":19108,"total_comment_lines":646}},{"id":1353675,"crate":"tokenizers","num":"0.20.4","dl_path":"/api/v1/crates/tokenizers/0.20.4/download","readme_path":"/api/v1/crates/tokenizers/0.20.4/readme","updated_at":"2024-11-26T17:03:22.255018Z","created_at":"2024-11-26T17:03:22.255018Z","downloads":1276059,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.20.4/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.20.4/downloads","authors":"/api/v1/crates/tokenizers/0.20.4/authors"},"crate_size":180130,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-11-26T17:03:22.255018Z"}],"checksum":"3b08cc37428a476fc9e20ac850132a513a2e1ce32b6a31addf2b74fa7033b905","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":19092,"comment_lines":646,"files":71}},"total_code_lines":19108,"total_comment_lines":646}},{"id":1328781,"crate":"tokenizers","num":"0.20.3","dl_path":"/api/v1/crates/tokenizers/0.20.3/download","readme_path":"/api/v1/crates/tokenizers/0.20.3/readme","updated_at":"2024-11-05T17:20:12.587698Z","created_at":"2024-11-05T17:20:12.587698Z","downloads":349728,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.20.3/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.20.3/downloads","authors":"/api/v1/crates/tokenizers/0.20.3/authors"},"crate_size":177293,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-11-05T17:20:12.587698Z"}],"checksum":"67b67c92f6d705e2a1d106fb0b28c696f9074901a9c656ee5d9f5de204c39bf7","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":18976,"comment_lines":646,"files":71}},"total_code_lines":18992,"total_comment_lines":646}},{"id":1327317,"crate":"tokenizers","num":"0.20.2","dl_path":"/api/v1/crates/tokenizers/0.20.2/download","readme_path":"/api/v1/crates/tokenizers/0.20.2/readme","updated_at":"2024-11-04T17:17:03.110954Z","created_at":"2024-11-04T17:17:03.110954Z","downloads":6515,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.20.2/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.20.2/downloads","authors":"/api/v1/crates/tokenizers/0.20.2/authors"},"crate_size":177278,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-11-04T17:17:03.110954Z"}],"checksum":"fef9821871de9277bd7d8a7568e798e775db7eede56e290b4b03f1dfb9ac6769","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":18976,"comment_lines":646,"files":71}},"total_code_lines":18992,"total_comment_lines":646}},{"id":1297016,"crate":"tokenizers","num":"0.20.1","dl_path":"/api/v1/crates/tokenizers/0.20.1/download","readme_path":"/api/v1/crates/tokenizers/0.20.1/readme","updated_at":"2024-10-10T09:46:24.127032Z","created_at":"2024-10-10T09:46:24.127032Z","downloads":93376,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.20.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.20.1/downloads","authors":"/api/v1/crates/tokenizers/0.20.1/authors"},"crate_size":175013,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-10-10T09:46:24.127032Z"}],"checksum":"b172ffa9a2e5c31bbddc940cd5725d933ced983a9333bbebc4c7eda3bbce1557","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":18941,"comment_lines":629,"files":71}},"total_code_lines":18957,"total_comment_lines":629}},{"id":1227941,"crate":"tokenizers","num":"0.20.0","dl_path":"/api/v1/crates/tokenizers/0.20.0/download","readme_path":"/api/v1/crates/tokenizers/0.20.0/readme","updated_at":"2024-08-08T16:45:14.911314Z","created_at":"2024-08-08T16:45:14.911314Z","downloads":89001,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.20.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.20.0/downloads","authors":"/api/v1/crates/tokenizers/0.20.0/authors"},"crate_size":174909,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-08-08T16:45:14.911314Z"}],"checksum":"c8a24d7f7d6be5b9d1377418b893ab1808af0074f5d1bb2c64784452ddd2aa70","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":18936,"comment_lines":629,"files":71}},"total_code_lines":18952,"total_comment_lines":629}},{"id":1118139,"crate":"tokenizers","num":"0.19.1","dl_path":"/api/v1/crates/tokenizers/0.19.1/download","readme_path":"/api/v1/crates/tokenizers/0.19.1/readme","updated_at":"2024-04-17T21:17:09.958279Z","created_at":"2024-04-17T21:17:09.958279Z","downloads":943716,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.19.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.19.1/downloads","authors":"/api/v1/crates/tokenizers/0.19.1/authors"},"crate_size":167036,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-04-17T21:17:09.958279Z"}],"checksum":"e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17919,"comment_lines":617,"files":70}},"total_code_lines":17935,"total_comment_lines":617}},{"id":1117627,"crate":"tokenizers","num":"0.19.0","dl_path":"/api/v1/crates/tokenizers/0.19.0/download","readme_path":"/api/v1/crates/tokenizers/0.19.0/readme","updated_at":"2024-04-17T08:52:21.506542Z","created_at":"2024-04-17T08:52:21.506542Z","downloads":12857,"features":{"default":["progressbar","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.19.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.19.0/downloads","authors":"/api/v1/crates/tokenizers/0.19.0/authors"},"crate_size":166758,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-04-17T08:52:21.506542Z"}],"checksum":"a3429c3955ff2d769edd699768796dafd1bbff0005ddca66882a1a9fce1b8c84","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17873,"comment_lines":617,"files":70}},"total_code_lines":17889,"total_comment_lines":617}},{"id":1048482,"crate":"tokenizers","num":"0.15.2","dl_path":"/api/v1/crates/tokenizers/0.15.2/download","readme_path":"/api/v1/crates/tokenizers/0.15.2/readme","updated_at":"2024-02-12T02:11:46.844759Z","created_at":"2024-02-12T02:11:46.844759Z","downloads":1411308,"features":{"cli":["clap"],"default":["progressbar","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.15.2/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.15.2/downloads","authors":"/api/v1/crates/tokenizers/0.15.2/authors"},"crate_size":167256,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-02-12T02:11:46.844759Z"}],"checksum":"3dd47962b0ba36e7fd33518fbf1754d136fd1474000162bbf2a8b5fcb2d3654d","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17803,"comment_lines":616,"files":71}},"total_code_lines":17819,"total_comment_lines":616}},{"id":1022543,"crate":"tokenizers","num":"0.15.1","dl_path":"/api/v1/crates/tokenizers/0.15.1/download","readme_path":"/api/v1/crates/tokenizers/0.15.1/readme","updated_at":"2024-01-22T15:01:56.337335Z","created_at":"2024-01-22T15:01:56.337335Z","downloads":25107,"features":{"cli":["clap"],"default":["progressbar","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.15.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.15.1/downloads","authors":"/api/v1/crates/tokenizers/0.15.1/authors"},"crate_size":166611,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2024-01-22T15:01:56.337335Z"}],"checksum":"6db445cceba5dfeb0f9702be7d6bfd91801ddcbe8fe8722defe7f2e96da75812","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17768,"comment_lines":597,"files":71}},"total_code_lines":17784,"total_comment_lines":597}},{"id":954784,"crate":"tokenizers","num":"0.15.0","dl_path":"/api/v1/crates/tokenizers/0.15.0/download","readme_path":"/api/v1/crates/tokenizers/0.15.0/readme","updated_at":"2023-11-14T18:51:49.962852Z","created_at":"2023-11-14T18:51:49.962852Z","downloads":112413,"features":{"cli":["clap"],"default":["progressbar","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.15.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.15.0/downloads","authors":"/api/v1/crates/tokenizers/0.15.0/authors"},"crate_size":166033,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2023-11-14T18:51:49.962852Z"}],"checksum":"062b8a9613d6017633b80fb55fbb33f1aff006c36225a3025630753398034b3c","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17694,"comment_lines":582,"files":71}},"total_code_lines":17710,"total_comment_lines":582}},{"id":917767,"crate":"tokenizers","num":"0.14.1","dl_path":"/api/v1/crates/tokenizers/0.14.1/download","readme_path":"/api/v1/crates/tokenizers/0.14.1/readme","updated_at":"2023-10-06T11:11:43.704463Z","created_at":"2023-10-06T11:11:43.704463Z","downloads":312777,"features":{"cli":["clap"],"default":["progressbar","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.14.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.14.1/downloads","authors":"/api/v1/crates/tokenizers/0.14.1/authors"},"crate_size":165243,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2023-10-06T11:11:43.704463Z"}],"checksum":"d9be88c795d8b9f9c4002b3a8f26a6d0876103a6f523b32ea3bac52d8560c17c","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17518,"comment_lines":582,"files":71}},"total_code_lines":17534,"total_comment_lines":582}},{"id":891526,"crate":"tokenizers","num":"0.14.0","dl_path":"/api/v1/crates/tokenizers/0.14.0/download","readme_path":"/api/v1/crates/tokenizers/0.14.0/readme","updated_at":"2023-09-07T14:00:57.772670Z","created_at":"2023-09-07T14:00:57.772670Z","downloads":9380,"features":{"cli":["clap"],"default":["progressbar","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.14.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.14.0/downloads","authors":"/api/v1/crates/tokenizers/0.14.0/authors"},"crate_size":166004,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2023-09-07T14:00:57.772670Z"}],"checksum":"12b515a66453a4d68f03398054f7204fd0dde6b93d3f20ea90b08025ab49b499","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17512,"comment_lines":582,"files":71}},"total_code_lines":17528,"total_comment_lines":582}},{"id":891416,"crate":"tokenizers","num":"0.14.0-rc.1","dl_path":"/api/v1/crates/tokenizers/0.14.0-rc.1/download","readme_path":"/api/v1/crates/tokenizers/0.14.0-rc.1/readme","updated_at":"2023-09-07T11:40:42.151879Z","created_at":"2023-09-07T11:40:42.151879Z","downloads":1173,"features":{"cli":["clap"],"default":["progressbar","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["hf-hub"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.14.0-rc.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.14.0-rc.1/downloads","authors":"/api/v1/crates/tokenizers/0.14.0-rc.1/authors"},"crate_size":166033,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2023-09-07T11:40:42.151879Z"}],"checksum":"2e8d61fbccf50b25a48e5bd7ca5f0ae92e3b55f4827931cf1b1cd02654118092","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17512,"comment_lines":582,"files":71}},"total_code_lines":17528,"total_comment_lines":582}},{"id":871541,"crate":"tokenizers","num":"0.13.4","dl_path":"/api/v1/crates/tokenizers/0.13.4/download","readme_path":"/api/v1/crates/tokenizers/0.13.4/readme","updated_at":"2023-08-14T11:47:58.620244Z","created_at":"2023-08-14T11:47:58.620244Z","downloads":423737,"features":{"cli":["clap"],"default":["progressbar","http","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["reqwest","cached-path","dirs"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.13.4/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.13.4/downloads","authors":"/api/v1/crates/tokenizers/0.13.4/authors"},"crate_size":169936,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2023-08-14T11:47:58.620244Z"}],"checksum":"aea68938177975ab09da68552b720eac941779ff386baceaf77e0f5f9cea645f","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17524,"comment_lines":581,"files":71}},"total_code_lines":17540,"total_comment_lines":581}},{"id":768694,"crate":"tokenizers","num":"0.13.3","dl_path":"/api/v1/crates/tokenizers/0.13.3/download","readme_path":"/api/v1/crates/tokenizers/0.13.3/readme","updated_at":"2023-04-05T08:05:25.784596Z","created_at":"2023-04-05T08:05:25.784596Z","downloads":127672,"features":{"cli":["clap"],"default":["progressbar","http","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["reqwest","cached-path","dirs"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.13.3/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.13.3/downloads","authors":"/api/v1/crates/tokenizers/0.13.3/authors"},"crate_size":166218,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2023-04-05T08:05:25.784596Z"}],"checksum":"5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":17159,"comment_lines":563,"files":71}},"total_code_lines":17175,"total_comment_lines":563}},{"id":658737,"crate":"tokenizers","num":"0.13.2","dl_path":"/api/v1/crates/tokenizers/0.13.2/download","readme_path":"/api/v1/crates/tokenizers/0.13.2/readme","updated_at":"2022-11-07T08:46:06.567778Z","created_at":"2022-11-07T08:46:06.567778Z","downloads":46503,"features":{"cli":["clap"],"default":["progressbar","http","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["reqwest","cached-path"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.13.2/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.13.2/downloads","authors":"/api/v1/crates/tokenizers/0.13.2/authors"},"crate_size":157456,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2022-11-07T08:46:06.567778Z"}],"checksum":"f4ff2dd291eac98dcea13e8cf7a0b28c373a90dc9210ccdab0fa9e69ee0cac69","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":16741,"comment_lines":560,"files":67}},"total_code_lines":16757,"total_comment_lines":560}},{"id":635649,"crate":"tokenizers","num":"0.13.1","dl_path":"/api/v1/crates/tokenizers/0.13.1/download","readme_path":"/api/v1/crates/tokenizers/0.13.1/readme","updated_at":"2022-10-06T13:12:44.532178Z","created_at":"2022-10-06T13:12:44.532178Z","downloads":2541,"features":{"cli":["clap"],"default":["progressbar","http","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["reqwest","cached-path"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.13.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.13.1/downloads","authors":"/api/v1/crates/tokenizers/0.13.1/authors"},"crate_size":157039,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2022-10-06T13:12:44.532178Z"}],"checksum":"3d7b08ede6742d7a59d58c71da8a6fa21bedc433dca2e855e439274d08df1170","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":16741,"comment_lines":560,"files":67}},"total_code_lines":16757,"total_comment_lines":560}},{"id":624218,"crate":"tokenizers","num":"0.13.0","dl_path":"/api/v1/crates/tokenizers/0.13.0/download","readme_path":"/api/v1/crates/tokenizers/0.13.0/readme","updated_at":"2022-09-19T08:16:05.101850Z","created_at":"2022-09-19T08:16:05.101850Z","downloads":5149,"features":{"cli":["clap"],"default":["progressbar","http","cli","onig","esaxx_fast"],"esaxx_fast":["esaxx-rs/cpp"],"http":["reqwest","cached-path"],"progressbar":["indicatif"],"unstable_wasm":["fancy-regex","getrandom/js"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.13.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.13.0/downloads","authors":"/api/v1/crates/tokenizers/0.13.0/authors"},"crate_size":157032,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2022-09-19T08:16:05.101850Z"}],"checksum":"72e10b4dcec4f6239730ce03b3c790f6d9ce1559bb5b263b9455436cdbea5b22","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":16738,"comment_lines":559,"files":67}},"total_code_lines":16754,"total_comment_lines":559}},{"id":524877,"crate":"tokenizers","num":"0.12.0","dl_path":"/api/v1/crates/tokenizers/0.12.0/download","readme_path":"/api/v1/crates/tokenizers/0.12.0/readme","updated_at":"2022-04-01T12:23:58.356757Z","created_at":"2022-03-31T09:13:05.165608Z","downloads":1164,"features":{"cli":["clap"],"default":["progressbar","http","cli"],"http":["reqwest","cached-path"],"progressbar":["indicatif"]},"yanked":true,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.12.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.12.0/downloads","authors":"/api/v1/crates/tokenizers/0.12.0/authors"},"crate_size":152660,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2022-03-31T09:13:05.165608Z"},{"action":"yank","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2022-04-01T12:23:58.360857Z"}],"checksum":"6048731c52227791c049be580b133d508957e1715dbb714abdbde10e8191419e","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":16118,"comment_lines":486,"files":63}},"total_code_lines":16134,"total_comment_lines":486}},{"id":507797,"crate":"tokenizers","num":"0.11.3","dl_path":"/api/v1/crates/tokenizers/0.11.3/download","readme_path":"/api/v1/crates/tokenizers/0.11.3/readme","updated_at":"2022-02-28T10:56:32.025826Z","created_at":"2022-02-28T10:56:32.025826Z","downloads":72643,"features":{"default":["progressbar","http"],"http":["reqwest","cached-path"],"progressbar":["indicatif"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.11.3/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.11.3/downloads","authors":"/api/v1/crates/tokenizers/0.11.3/authors"},"crate_size":151489,"published_by":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"audit_actions":[{"action":"publish","user":{"id":79201,"login":"Narsil","name":"Nicolas Patry","avatar":"https://avatars.githubusercontent.com/u/204321?v=4","url":"https://github.com/Narsil","github_username_matches":true,"created_at":"2010-02-16T01:56:35Z"},"time":"2022-02-28T10:56:32.025826Z"}],"checksum":"60c79b46b9e4a7c8a4926531d1b81521a976cab19f1651627385504aa68799f2","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":16068,"comment_lines":483,"files":63}},"total_code_lines":16084,"total_comment_lines":483}},{"id":420113,"crate":"tokenizers","num":"0.11.0","dl_path":"/api/v1/crates/tokenizers/0.11.0/download","readme_path":"/api/v1/crates/tokenizers/0.11.0/readme","updated_at":"2021-08-31T14:57:14.331649Z","created_at":"2021-08-31T14:57:14.331649Z","downloads":8554,"features":{"default":["progressbar"],"progressbar":["indicatif"]},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.11.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.11.0/downloads","authors":"/api/v1/crates/tokenizers/0.11.0/authors"},"crate_size":148182,"published_by":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"audit_actions":[{"action":"publish","user":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"time":"2021-08-31T14:57:14.331649Z"}],"checksum":"6f5c159c4c2efb7d3fbed139b726cb5c0135b12d68e2c09afc1a851f8f2534e8","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Pan":{"code_lines":16,"comment_lines":0,"files":1},"Rust":{"code_lines":15599,"comment_lines":469,"files":63}},"total_code_lines":15615,"total_comment_lines":469}},{"id":229345,"crate":"tokenizers","num":"0.10.1","dl_path":"/api/v1/crates/tokenizers/0.10.1/download","readme_path":"/api/v1/crates/tokenizers/0.10.1/readme","updated_at":"2020-04-09T15:38:17.318143Z","created_at":"2020-04-09T15:38:17.318143Z","downloads":18660,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.10.1/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.10.1/downloads","authors":"/api/v1/crates/tokenizers/0.10.1/authors"},"crate_size":56755,"published_by":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"audit_actions":[{"action":"publish","user":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"time":"2020-04-09T15:38:17.318143Z"}],"checksum":"3632a2635700a4b46f875eba3d6bf83c48c3c4289aa6330442a9ed8e3dcf587c","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":5215,"comment_lines":198,"files":35}},"total_code_lines":5215,"total_comment_lines":198}},{"id":229036,"crate":"tokenizers","num":"0.10.0","dl_path":"/api/v1/crates/tokenizers/0.10.0/download","readme_path":"/api/v1/crates/tokenizers/0.10.0/readme","updated_at":"2020-04-08T18:23:49.814546Z","created_at":"2020-04-08T18:23:49.814546Z","downloads":1322,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.10.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.10.0/downloads","authors":"/api/v1/crates/tokenizers/0.10.0/authors"},"crate_size":57235,"published_by":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"audit_actions":[{"action":"publish","user":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"time":"2020-04-08T18:23:49.814546Z"}],"checksum":"be8b65c3eaa8bf0a77f2c9ebca4ee5144b95efe964b3d606f5998347ae2da29a","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":5228,"comment_lines":198,"files":35}},"total_code_lines":5228,"total_comment_lines":198}},{"id":226087,"crate":"tokenizers","num":"0.9.0","dl_path":"/api/v1/crates/tokenizers/0.9.0/download","readme_path":"/api/v1/crates/tokenizers/0.9.0/readme","updated_at":"2020-03-30T16:33:50.011811Z","created_at":"2020-03-30T16:33:50.011811Z","downloads":1382,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.9.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.9.0/downloads","authors":"/api/v1/crates/tokenizers/0.9.0/authors"},"crate_size":56629,"published_by":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"audit_actions":[{"action":"publish","user":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"time":"2020-03-30T16:33:50.011811Z"}],"checksum":"a0fd68843c058037cefdf90aa1fb8399a3a35871d3d2260567c957169b4b8b4e","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":5134,"comment_lines":198,"files":35}},"total_code_lines":5134,"total_comment_lines":198}},{"id":217458,"crate":"tokenizers","num":"0.8.0","dl_path":"/api/v1/crates/tokenizers/0.8.0/download","readme_path":"/api/v1/crates/tokenizers/0.8.0/readme","updated_at":"2020-03-02T19:52:02.754781Z","created_at":"2020-03-02T19:52:02.754781Z","downloads":1391,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.8.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.8.0/downloads","authors":"/api/v1/crates/tokenizers/0.8.0/authors"},"crate_size":48024,"published_by":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"audit_actions":[{"action":"publish","user":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"time":"2020-03-02T19:52:02.754781Z"}],"checksum":"b3833c588cc6992a6f51583ec74bddf3198d48abcf0e77b785e863877d631775","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":4444,"comment_lines":185,"files":32}},"total_code_lines":4444,"total_comment_lines":185}},{"id":209768,"crate":"tokenizers","num":"0.7.0","dl_path":"/api/v1/crates/tokenizers/0.7.0/download","readme_path":"/api/v1/crates/tokenizers/0.7.0/readme","updated_at":"2020-02-05T19:14:12.772872Z","created_at":"2020-02-05T19:14:12.772872Z","downloads":1366,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.7.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.7.0/downloads","authors":"/api/v1/crates/tokenizers/0.7.0/authors"},"crate_size":44784,"published_by":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"audit_actions":[{"action":"publish","user":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"time":"2020-02-05T19:14:12.772872Z"}],"checksum":"9dc9cd60edeba70cc6d45c23ba56e82746c74e838ffecbca6ab693c7e7ab309c","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":3954,"comment_lines":167,"files":31}},"total_code_lines":3954,"total_comment_lines":167}},{"id":202831,"crate":"tokenizers","num":"0.6.0","dl_path":"/api/v1/crates/tokenizers/0.6.0/download","readme_path":"/api/v1/crates/tokenizers/0.6.0/readme","updated_at":"2020-01-10T17:30:25.044704Z","created_at":"2020-01-10T17:30:25.044704Z","downloads":1394,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.6.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.6.0/downloads","authors":"/api/v1/crates/tokenizers/0.6.0/authors"},"crate_size":40641,"published_by":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"audit_actions":[{"action":"publish","user":{"id":70987,"login":"n1t0","name":"Anthony MOI","avatar":"https://avatars.githubusercontent.com/u/1217986?v=4","url":"https://github.com/n1t0","github_username_matches":true,"created_at":"2011-11-24T11:55:50Z"},"time":"2020-01-10T17:30:25.044704Z"}],"checksum":"9433b9ceccfae0806695f15660a69f5bdba80ee63add167a0ff3040f521d12e3","rust_version":null,"has_lib":true,"bin_names":["cli"],"edition":"2018","description":"Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.","homepage":"https://github.com/huggingface/tokenizers","documentation":"https://docs.rs/tokenizers/","repository":"https://github.com/huggingface/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":3473,"comment_lines":157,"files":28}},"total_code_lines":3473,"total_comment_lines":157}},{"id":181582,"crate":"tokenizers","num":"0.5.0","dl_path":"/api/v1/crates/tokenizers/0.5.0/download","readme_path":"/api/v1/crates/tokenizers/0.5.0/readme","updated_at":"2019-10-09T22:12:25.634083Z","created_at":"2019-10-09T22:12:25.634083Z","downloads":1403,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"MIT/Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.5.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.5.0/downloads","authors":"/api/v1/crates/tokenizers/0.5.0/authors"},"crate_size":8937,"published_by":{"id":55850,"login":"epwalsh","name":"Pete Walsh","avatar":"https://avatars.githubusercontent.com/u/8812459?v=4","url":"https://github.com/epwalsh","github_username_matches":true,"created_at":"2014-09-17T21:22:10Z"},"audit_actions":[],"checksum":"53a274405313983f0e3b5d59b666c5d8cf6c2150b2466a917a694176a221336e","rust_version":null,"has_lib":true,"bin_names":[],"edition":"2018","description":"Provides a generic Tokenizer trait and a smart Token struct","homepage":"https://github.com/epwalsh/tokenizers","documentation":null,"repository":"https://github.com/epwalsh/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":97,"comment_lines":0,"files":1}},"total_code_lines":97,"total_comment_lines":0}},{"id":168405,"crate":"tokenizers","num":"0.2.0","dl_path":"/api/v1/crates/tokenizers/0.2.0/download","readme_path":"/api/v1/crates/tokenizers/0.2.0/readme","updated_at":"2019-08-09T16:55:13.142267Z","created_at":"2019-08-09T16:55:13.142267Z","downloads":1365,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"MIT/Apache-2.0","links":{"dependencies":"/api/v1/crates/tokenizers/0.2.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.2.0/downloads","authors":"/api/v1/crates/tokenizers/0.2.0/authors"},"crate_size":4707,"published_by":{"id":55850,"login":"epwalsh","name":"Pete Walsh","avatar":"https://avatars.githubusercontent.com/u/8812459?v=4","url":"https://github.com/epwalsh","github_username_matches":true,"created_at":"2014-09-17T21:22:10Z"},"audit_actions":[],"checksum":"221c8fbda5bb953274362eedd9a43e3263f305c7344172ef522b2ff0586d4683","rust_version":null,"has_lib":true,"bin_names":["tokenize"],"edition":"2018","description":"Lightweight tokenizers with a simple CLI","homepage":"https://github.com/epwalsh/tokenizers","documentation":null,"repository":"https://github.com/epwalsh/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":216,"comment_lines":4,"files":5}},"total_code_lines":216,"total_comment_lines":4}},{"id":168162,"crate":"tokenizers","num":"0.1.0","dl_path":"/api/v1/crates/tokenizers/0.1.0/download","readme_path":"/api/v1/crates/tokenizers/0.1.0/readme","updated_at":"2019-08-08T14:55:49.017781Z","created_at":"2019-08-08T14:55:49.017781Z","downloads":1374,"features":{},"yanked":false,"yank_message":null,"lib_links":null,"license":"MIT","links":{"dependencies":"/api/v1/crates/tokenizers/0.1.0/dependencies","version_downloads":"/api/v1/crates/tokenizers/0.1.0/downloads","authors":"/api/v1/crates/tokenizers/0.1.0/authors"},"crate_size":4883,"published_by":{"id":55850,"login":"epwalsh","name":"Pete Walsh","avatar":"https://avatars.githubusercontent.com/u/8812459?v=4","url":"https://github.com/epwalsh","github_username_matches":true,"created_at":"2014-09-17T21:22:10Z"},"audit_actions":[],"checksum":"7a21deecd3b65ee874286d0f76927258ef83d40dcb4138c402c3edb96a551dd5","rust_version":null,"has_lib":true,"bin_names":["tokenize"],"edition":"2018","description":"Lightweight tokenizers with a simple CLI","homepage":"https://github.com/epwalsh/tokenizers","documentation":null,"repository":"https://github.com/epwalsh/tokenizers","trustpub_data":null,"linecounts":{"languages":{"Rust":{"code_lines":309,"comment_lines":11,"files":4}},"total_code_lines":309,"total_comment_lines":11}}],"keywords":[{"id":"bpe","keyword":"bpe","created_at":"2020-01-10T17:30:25.044704Z","crates_cnt":66},{"id":"huggingface","keyword":"huggingface","created_at":"2020-01-10T17:30:25.044704Z","crates_cnt":61},{"id":"nlp","keyword":"nlp","created_at":"2015-03-20T14:53:14.632874Z","crates_cnt":686},{"id":"tokenizer","keyword":"tokenizer","created_at":"2015-02-21T23:25:36.408739Z","crates_cnt":312},{"id":"wordpiece","keyword":"wordpiece","created_at":"2019-11-29T07:56:17.387456Z","crates_cnt":9}],"categories":[]}