llmcompressor.transformers.finetune.data.c4

`C4Dataset`

Bases: TextGenerationDataset

Child text generation class for the C4 dataset

Parameters:

Name	Type	Description	Default
`dataset_args`	`DatasetArguments`	configuration settings for dataset loading	required
`split`	`str`	split from dataset to load, for instance `test` or `train[:5%]`	required
`processor`	`Processor`	processor or tokenizer to use on dataset	required

Source code in llmcompressor/transformers/finetune/data/c4.py

@TextGenerationDataset.register(name="c4")
class C4Dataset(TextGenerationDataset):
    """
    Child text generation class for the C4 dataset

    :param dataset_args: configuration settings for dataset loading
    :param split: split from dataset to load, for instance `test` or `train[:5%]`
    :param processor: processor or tokenizer to use on dataset
    """

    def __init__(
        self, dataset_args: "DatasetArguments", split: str, processor: Processor
    ):
        dataset_args = deepcopy(dataset_args)
        dataset_args.dataset = "allenai/c4"
        dataset_args.text_column = "text"

        super().__init__(dataset_args=dataset_args, split=split, processor=processor)