trinity.buffer.reader.file_reader module

trinity.buffer.reader.file_reader module#

Filed based buffer reader.

class trinity.buffer.reader.file_reader.DummyProgressBar[source]#

Bases: object

__init__()[source]#
update(num: int)[source]#
close()[source]#
class trinity.buffer.reader.file_reader.BaseFileReader[source]#

Bases: BufferReader

async read_async(batch_size: int | None = None, **kwargs)[source]#

Read from buffer asynchronously.

class trinity.buffer.reader.file_reader.FileReader(config: StorageConfig)[source]#

Bases: BaseFileReader

Provide a unified interface for Experience and Task file readers.

__init__(config: StorageConfig)[source]#
read(batch_size: int | None = None, **kwargs) → List[source]#

Read from buffer.

state_dict()[source]#

Return the state of the reader as a dict. :returns: A dict containing the reader state. At minimum, it should contain

the current_index field.

load_state_dict(state_dict)[source]#
feedback(**pipeline_metrics)[source]#
class trinity.buffer.reader.file_reader.ExperienceFileReader(config: StorageConfig)[source]#

Bases: _DatasetFileReader

Reader for SFT / DPO file data.

__init__(config: StorageConfig)[source]#
class trinity.buffer.reader.file_reader.TaskFileReader(config: StorageConfig)[source]#

Bases: _DatasetFileReader

A Reader for task file data.

__init__(config: StorageConfig)[source]#