trainer:
  nnodes: 1
  n_gpus_per_node: 8

data:
  path: ~/data/rlhf/math/test.parquet
  prompt_key: prompt
  response_key: responses
  data_source_key: data_source
  reward_model_key: reward_model
  n_samples: 1
  output_path: /opt/tiger/math_Qwen2-7B-Instruct.parquet
  batch_size: 2048

model:
  path: ~/models/Qwen2-7B-Instruct
  external_lib: null

rollout:
  name: vllm
  temperature: 0.6
  top_k: -1 # 0 for hf rollout, -1 for vllm rollout
  top_p: 0.95
  prompt_length: 2048
  response_length: 32768
  # for vllm rollout
  dtype: bfloat16 # should align with FSDP
  gpu_memory_utilization: 0.9
  ignore_eos: False
  micro_batch_size: 256
  enforce_eager: True
  free_cache_engine: True
  load_format: dummy_dtensor
  tensor_model_parallel_size: 1
  max_num_batched_tokens: 8192
  max_num_seqs: 1024
  log_prob_micro_batch_size: 8
  # for hf rollout
  do_sample: True
  # This must be set to one for eval script to work.
  # Use data.num_samples instead.
  n: 1
  n_val: 1
  enable_chunked_prefill: True
  e1_thinking_length: 1024
  e1_solution_length: 1024

# Unneccessary parameters for generation.
# Just for properly instantiating FSDP workers.
actor:
  strategy: fsdp  # This is for backward-compatibility
  ulysses_sequence_parallel_size: 1 # sp size
  fsdp_config:
    wrap_policy:
      # transformer_layer_cls_to_wrap: None
      min_num_params: 0
    param_offload: False
    grad_offload: False
    optimizer_offload: False
    fsdp_size: -1
  ppo_mini_batch_size: 256
  ppo_micro_batch_size: 64
  use_dynamic_bsz: False
  ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length}
  optim:
    lr: 1e-6
    lr_warmup_steps_ratio: 0.  # the total steps will be injected during runtime
    min_lr_ratio: null   # only useful for warmup with cosine
    warmup_style: constant  # select from constant/cosine
    total_training_steps: -1  # must be override by program