preprocess and tokenize datasets

[tyoc213]

Rewrite of the work done in #43

[vishalbakshi]

TBD if we need to do this or not

@tyoc213 we should replace this:

    def __iter__(self) -> Iterable[dict[str, NDArray]]:
        buffer = []
        for sample in self.hf_dataset:
            encoded = self.tokenizer(
                sample['text'],
                truncation=False,
                padding=False,
            )
            iids = encoded['input_ids']
            buffer = buffer + self.bos_tokens + iids + self.eos_tokens
            while len(buffer) >= self.max_length:
                concat_sample = buffer[:self.max_length]
                buffer = buffer[self.max_length:] if self.should_wrap else []
                yield {
                    # convert to ndarray to store in MDS format
                    'tokens': np.asarray(concat_sample, dtype=np.int32),
                }

to something like this:

    def __iter__(self) -> Iterable[dict[str, NDArray]]:
        for sample in self.hf_dataset:
            encoded = self.tokenizer(
                sample['text'],
                truncation=False,
                padding=False,
            )
            iids = self.bos_tokens + encoded['input_ids'] + self.eos_tokens
            yield {
                # convert to ndarray to store in MDS format
                'tokens': np.asarray(iids[:self.max_length],  dtype=np.int32),
            }