BDEPEND=test? ( sci-ml/datasets[python_single_target_python3_12(-)?,python_single_target_python3_13(-)?] ) python_single_target_python3_12? ( dev-python/setuptools-rust[python_targets_python3_12(-)] ) python_single_target_python3_13? ( dev-python/setuptools-rust[python_targets_python3_13(-)] ) test? ( dev-libs/oniguruma python_single_target_python3_12? ( >=dev-python/pytest-7.4.4[python_targets_python3_12(-)] ) python_single_target_python3_13? ( >=dev-python/pytest-7.4.4[python_targets_python3_13(-)] ) ) || ( >=dev-lang/rust-bin-1.82.0:* >=dev-lang/rust-1.82.0:* ) python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 ) python_single_target_python3_12? ( >=dev-python/gpep517-16[python_targets_python3_12(-)] >=dev-util/maturin-1.8.2[python_targets_python3_12(-)] ) python_single_target_python3_13? ( >=dev-python/gpep517-16[python_targets_python3_13(-)] >=dev-util/maturin-1.8.2[python_targets_python3_13(-)] )
DEFINED_PHASES=compile configure install prepare setup test unpack
DEPEND=python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 )
DESCRIPTION=Implementation of today's most used tokenizers
EAPI=8
HOMEPAGE=https://github.com/huggingface/tokenizers
INHERIT=cargo distutils-r1
IUSE=test debug python_single_target_python3_12 python_single_target_python3_13 debug
KEYWORDS=~amd64
LICENSE=Apache-2.0 Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD-2 BSD ISC MIT MPL-2.0 Unicode-DFS-2016
RDEPEND=dev-libs/oniguruma python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 )
REQUIRED_USE=^^ ( python_single_target_python3_12 python_single_target_python3_13 )
RESTRICT=!test? ( test )
SLOT=0
SRC_URI=https://github.com/huggingface/tokenizers/archive/refs/tags/v0.22.1.tar.gz -> tokenizers-0.22.1.gh.tar.gz https://dev.gentoo.org/~tupone/distfiles/tokenizers-0.22.0-crates.tar.xz https://dev.gentoo.org/~tupone/distfiles/tokenizers-python-0.22.1-crates.tar.xz
_eclasses_=cargo	53bab5a62cfc3526d8ed02c0a38a6033	distutils-r1	62add15ed646bfe511af53e282621b39	eapi9-pipestatus	d2c134036ac31c3085aebc9147f572bd	flag-o-matic	fd3558f73f6503093adee69adf41020d	multibuild	83db098a835f71feb71761f72a694aa2	multilib	fc812e8eb2170022c28272d80a1f9e77	multiprocessing	45d0620f25c2f063ad6276ecd9ed9399	ninja-utils	d063ea1900a793f93eb095010d6d9156	out-of-source-utils	8e5093bd27b7f541a1913271f25d633a	python-single-r1	47d8ac9be330c9366112b78fd4f8ce57	python-utils-r1	9c4180f103a0e8340bc993fc85dbdadd	rust	c5ae4c9b5eca97bd7fc77fede663873f	rust-toolchain	76468983281b0a7fc167ca224f84ecfd	toolchain-funcs	5195689ff6a73b0e789acfa09d4fbcb9
_md5_=fee1d5b087058504cab3412a7c696171