#!/bin/bash
# shellcheck disable=SC2090,SC2086,SC2089,SC1091
# Default project path
PROJECT_PATH="$HOME/projects/mt_dao"

# Parse command-line options
if ! OPTIONS=$(getopt -o p: --long project_path: -n 'parse-options' -- "$@"); then
	echo "Error parsing options" >&2
	exit 1
fi

eval set -- "$OPTIONS"

while true; do
	case "$1" in
	-p | --project_path)
		PROJECT_PATH="$2"
		shift 2
		;;
	--)
		shift
		break
		;;
	*)
		break
		;;
	esac
done

echo "PROJECT_PATH=$PROJECT_PATH"
#! Moving to the project folder
cd "$PROJECT_PATH" || exit
#! Preparing environment
if [[ $(hostname) == *'gpu-q'* ]]; then
	echo "Assuming the script is executing in the CSD3."
	#! Executing the environment preparation script
	#! NOTE: Must use "." to execute, "sh" doesn't work
	. "$PROJECT_PATH"/scripts/install_hpc_env.sh
else
	echo "Assuming the script is executing NOT in the CSD3."
	#! Executing the environment preparation script
	#! NOTE: Must use "." to execute, "sh" doesn't work
	. "$PROJECT_PATH"/scripts/install_env.sh
fi
#! Export the endpoint of the S3 object store, using directly the IP to avoid name resolution issues
export S3_ENDPOINT_URL='http://128.232.115.0:9000'

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name sr \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240919093538_v-50257_l-2048_d-allenai-c4_n-sr_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name la \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240919000759_v-50257_l-2048_d-allenai-c4_n-la_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name sw \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240918233908_v-50257_l-2048_d-allenai-c4_n-sw_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name it \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240921115318_v-50257_l-2048_d-allenai-c4_n-it_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name de \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240921120851_v-50257_l-2048_d-allenai-c4_n-de_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name ru \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240921181300_v-50257_l-2048_d-allenai-c4_n-ru_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name hi \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240921172445_v-50257_l-2048_d-allenai-c4_n-hi_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name el \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240921173321_v-50257_l-2048_d-allenai-c4_n-el_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name ur \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240921194442_v-50257_l-2048_d-allenai-c4_n-ur_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name es \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240921200055_v-50257_l-2048_d-allenai-c4_n-es_s-train \
	--num_clients 8

#! Execute the command
uv run python -m mt_dao.dataset.convert_dataset_hf \
	--path "allenai/c4" \
	--name ms \
	--splits validation train \
	--tokenizer /nfs-share/anon/projects/mt_dao/tokenizer_20240921222137_v-50257_l-2048_d-allenai-c4_n-ms_s-train \
	--num_clients 8

#! Remove the positional arguments
eval set --
