#!/bin/bash
# Uses https://github.com/google-research/google-research/blob/master/mbpp/sanitized-mbpp.json
set -e

OUTPUT_DIR="results/mbpp"
mkdir -p "$OUTPUT_DIR"

echo "=============================="
echo " MBPP (sanitized): TMPC Generation"
echo "=============================="
python run_mbpp.py \
    --output_folder "$OUTPUT_DIR/generation" \
    --max_iterations 5 \
    --num_candidates 3 \
    --load_in_4bit \
    --cuda_num 0 \
    --start 0 \
    --end 427

echo "=============================="
echo " MBPP: Evaluation"
echo "=============================="
python run_mbpp.py \
    --evaluate \
    --eval_folder "$OUTPUT_DIR/generation" \
    --eval_max_iter 5 \
    --eval_output "$OUTPUT_DIR/eval_results.csv" \
    --eval_from_folder \
    --eval_start 11 \
    --eval_range 510

echo "MBPP experiment complete. Results saved to $OUTPUT_DIR/"
