-- extends 'datasets/dataset_type.yaml'

-- from 'inc/formatting.jinja' import h3

-- block config_metadata
    == super()
    -- set ns.config_name = "Undefined"
    -- set ns.config_description = ""
    -- set ns.source = ""
    -- set ns.main_feature = "text"
-- endblock config_metadata

    

-- block main_body
== h3('Tokenizer')

    -- filter trim()
    -- block tokenizer
tokenizer: &tokenizer !var "tokenizer"
    << endblock tokenizer
    -- endfilter


    -- filter trim()
    -- block map_function
    << endblock map_function
    -- endfilter


    -- filter trim()
    -- block map_kwargs
    << endblock map_kwargs
    -- endfilter


    -- block datasets
        -- filter trim()
        -- block dataset_splits

== h3('Dataset Dictionaries')
            -- filter trim()
            -- block dataset_dict
            << endblock dataset_dict
            -- endfilter


== h3('Dataset Splits')
## Notes on split:
## Including the splits here is optional, although doing so makes it possible for the CLI
## to sample directly from the splits, without having to tokenize anything.
##
## Naming Conventions:
##   train: The main training dataset
##   validation: Samples held-out from the model for the purpose of hypter-parameter tuning,
##       measuring over/underfit, etc.
##   test: Additional helt-out samples, which should only be used for testin the final model.
##   eval: The dataset used by the trainer to measure performance. This should be 'validaiton,' for 
##       periodic testing, while training, and 'test,' for testing the final model.

            -- filter trim()
            -- block train_dataset_split
## train_dataset_split: &train_dataset_split
            << endblock train_dataset_split
            -- endfilter


            -- filter trim()
            -- block validation_dataset_split
## validation_dataset_split: &validation_dataset_split
            << endblock validation_dataset_split
            -- endfilter


            -- filter trim()
            -- block test_dataset_split
## test_dataset_split: &test_dataset_split
            << endblock test_dataset_split
            -- endfilter


        << endblock dataset_splits
        -- endfilter


== h3('Preprocessed Datasets')

        -- filter trim()
        -- block preprocessed_datasets
        
            -- filter trim()
            -- block train_dataset required
##train_dataset: &train_dataset
            << endblock train_dataset
            -- endfilter


            -- filter trim()
            -- block eval_dataset required
##eval_dataset: &eval_dataset
            << endblock eval_dataset
            -- endfilter


            -- filter trim()
            -- block test_dataset required
##test_dataset: &test_dataset
            << endblock test_dataset
            -- endfilter


        << endblock preprocessed_datasets
        -- endfilter
    << endblock datasets

<< endblock main_body
