Untitled
unknown
plain_text
9 months ago
13 kB
14
Indexable
#!/bin/sh
# set -e # Stop on any error
# 自动检测 Conda 安装路径并加载 conda.sh
set -e # 启用错误退出，确保出错时整个脚本退出
if [ -f "$HOME/anaconda3/etc/profile.d/conda.sh" ]; then
. "$HOME/anaconda3/etc/profile.d/conda.sh"
elif [ -f "$HOME/miniconda3/etc/profile.d/conda.sh" ]; then
. "$HOME/miniconda3/etc/profile.d/conda.sh"
fi
# conda init
# . ~/.bashrc # 或者 . ~/.zshrc，取决于你使用的 shell
timestamp=$(date '+%Y%m%d_%H%M%S')
# 使用第一个参数作为 model_name，如果没有提供则使用默认值
model_name="${1:-gpt-4.1}"
echo "模型名称: $model_name"
EVAL_PREPARE=false
FINAL_EVAL=false
GET_RESULTS=false
if [[ "$3" == "eval_prepare" ]]; then
echo "eval_prepare is set to true, skipping all phases except file_prepare."
EVAL_PREPARE=true
elif [[ "$3" == "generate" ]]; then
echo "generate is set to true, skipping all phases except start_eval."
GENERATE=true
elif [[ "$3" == "get_results" ]]; then
echo "get_results is set to true, skipping all phases except get_results."
GET_RESULTS=true
elif [[ "$3" == "final_eval" ]]; then
echo "final_eval is set to true, skipping all phases except final_eval."
FINAL_EVAL=true
elif [[ "$3" == "all" ]]; then
echo "all is set to true, running all phases."
EVAL_PREPARE=true
GENERATE=true
GET_RESULTS=true
FINAL_EVAL=true
fi
backend_name="${6:-OpenAI}"
dataset_name="${5:-princeton-nlp/SWE-bench_Verified}"
dataset_name_processed=$(echo "$dataset_name" | awk -F'/' '{print $NF}')
test_version="${2:-v0}" # Set this to the actual test version you want to use
max_content_length="${4:-128000}"
echo "max_content_length: $max_content_length"
start_index="${7:-0}"
end_index="${8:-100}"
generate_num_threads="${9:-12}"
execute_num_threads="${10:-20}"
if [ "$test_version" == "v0" ]; then
dataset_name_short="$(echo "$dataset_name_processed" | tr '_' '-' | tr '[:upper:]' '[:lower:]')"
else
dataset_name_short="$(echo "$dataset_name_processed" | tr '_' '-' | tr '[:upper:]' '[:lower:]')_${test_version}"
fi
echo "Dataset name: $dataset_name_short"
# Set dataset_num based on test_version
case $dataset_name_short in
"swe-bench-verified")
dataset_num=500
project_file_loc='repo_structure/repo_structures'
eval_path="SWE-bench"
eval_conda="swebench"
;;
"swe-bench-verified_v1")
dataset_num=200
project_file_loc='repo_structure/repo_structures'
eval_path="SWE-bench"
eval_conda="swebench"
;;
"swe-bench-verified_v2C")
dataset_num=100
project_file_loc='repo_structure/repo_structures'
eval_path="SWE-bench"
eval_conda="swebench"
;;
"swe-bench-verified_v11C")
dataset_num=55
project_file_loc='repo_structure/repo_structures'
eval_path="SWE-bench"
eval_conda="swebench"
;;
"swe-bench-verified_v21C")
dataset_num=50
project_file_loc='repo_structure/repo_structures'
eval_path="SWE-bench"
eval_conda="swebench"
;;
"swe-bench-live")
dataset_num=1565
project_file_loc='repo_structure/live_repo'
eval_path="SWE-bench-Live"
eval_conda="swebench-live"
;;
"swe-bench-live_v2l")
dataset_num=111
project_file_loc='repo_structure/live_repo'
eval_path="SWE-bench-Live"
eval_conda="swebench-live"
;;
"swe-bench-live_v2L")
dataset_num=100
project_file_loc='repo_structure/live_repo'
eval_path="SWE-bench-Live"
eval_conda="swebench-live"
;;
"swe-bench-live_v11L")
dataset_num=70
project_file_loc='repo_structure/live_repo'
eval_path="SWE-bench-Live"
eval_conda="swebench-live"
;;
*)
echo "Invalid test version"
exit 1
;;
esac
echo "Using test version: $test_version with dataset size: $dataset_num, project file location: $project_file_loc"
conda activate cc_eval
cd $(dirname "$0")
echo "Current conda environment: $CONDA_DEFAULT_ENV"
echo "Current working directory: $(pwd)"
export PYTHONPATH=$PYTHONPATH:$(pwd)
export PROJECT_FILE_LOC=${project_file_loc}
############################
# Phase 1: file_prepare
############################
if [ "$EVAL_PREPARE" = true ]; then
log_file="results/${model_name}/${dataset_name_short}/file_prepare/file_prepare_${timestamp}.log"
mkdir -p "results/${model_name}/${dataset_name_short}/file_prepare"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Phase 1 (file_prepare) starting, log file: $(pwd)/${log_file}"
touch "${log_file}"
python file_prepare.py \
--model_name $model_name \
--dataset_name_short $dataset_name_short \
--test_version $test_version \
--start_index $start_index \
--end_index $end_index \
--backend $backend_name \
--dataset $dataset_name > $log_file 2>&1
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Phase 1 (file_prepare) complete"
fi
############################
# Phase 2: docker_prepare
############################
if [ "$EVAL_PREPARE" = true ]; then
echo -n > port_map.txt
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Phase 2 (docker_prepare) starting"
mapfile -t test_instances < <(jq -r '.[]' "test_version_index/${test_version}.json" | sort)
BASE_PORT="${CLIENT_BASE_PORT:-40000}"
for (( i=${start_index}; i < ${end_index}; i++ )); do
instance_id=${test_instances[i]}
if [ -f "results/${model_name}/${dataset_name_short}/playground/$instance_id/workspace/finished.txt" ]; then
echo "Skipping $instance_id as finished.txt is present."
continue
fi
echo "instance_id: $instance_id"
if [ "$(docker ps -q -f name=ccr_$instance_id_$model_name)" ]; then
docker rm -f ccr_${instance_id}_${model_name}
fi
echo "start docker"
chmod +x results/${model_name}/${dataset_name_short}/playground/$instance_id/workspace/run.sh
BASE_IMAGE_NAME="sweb.eval.x86_64.${instance_id}"
TARGET_IMAGE_NAME="cc_sweb.eval.x86_64.${instance_id}"
WORKSPACE_DIR="$(pwd)/results/${model_name}/${dataset_name_short}/playground/${instance_id}/workspace"
DOCKERFILE_PATH="${WORKSPACE_DIR}/Dockerfile"
# Step 1: Check if the target Docker image exists
# We use `docker images -q` which returns the image ID if it exists, and an empty string otherwise.
if [ -z "$(docker images -q ${TARGET_IMAGE_NAME})" ]; then
echo "--> Image '${TARGET_IMAGE_NAME}' not found. Proceeding with build process."
# Step 1a: Check if the Dockerfile exists before trying to modify it
if [ ! -f "${DOCKERFILE_PATH}" ]; then
echo "--> ERROR: Dockerfile not found at '${DOCKERFILE_PATH}'. Cannot build."
exit 1
fi
# Step 1b: Modify the first line of the Dockerfile to change the base image
# echo "--> Modifying Dockerfile to use base image: '${BASE_IMAGE_NAME}'"
# Replace the first occurence of FROM ... with the proper base image name.
# We use `|` as a separator to avoid conflicts if the image name contained slashes.
# sed "0,/^FROM .*/{s|^FROM .*|FROM ${BASE_IMAGE_NAME}|;}" "${DOCKERFILE_PATH}" > "${DOCKERFILE_PATH}.tmp" && \
# mv "${DOCKERFILE_PATH}.tmp" "${DOCKERFILE_PATH}"
# Step 1c: Build the Docker image
echo "--> Building image '${TARGET_IMAGE_NAME}' using context path '${WORKSPACE_DIR}'......"
echo "--> Build log: ${WORKSPACE_DIR}/docker_build.log"
docker build -t "${TARGET_IMAGE_NAME}" "${WORKSPACE_DIR}" > ${WORKSPACE_DIR}/docker_build.log 2>&1
echo "--> Build complete."
else
echo "--> Image '${TARGET_IMAGE_NAME}' already exists. Skipping build."
fi
# Step 2: Run the Docker container
# This part runs regardless of whether the image was just built or already existed.
echo "--> Starting container 'ccr_${instance_id}_${model_name}'..."
docker run --rm -d --name "ccr_${instance_id}_${model_name}" \
-v "${WORKSPACE_DIR}:/workspace" \
-p 8730 \
"${TARGET_IMAGE_NAME}" \
bash -c "/workspace/run.sh"
host_port=$(docker port "ccr_${instance_id}_${model_name}" 8730 | sed 's/.*://' | head -n 1)
echo "$i, $host_port" >> port_map.txt
echo "--> Container started successfully."
echo "output file path: $(pwd)/results/${model_name}/${dataset_name_short}/playground/$instance_id/workspace/output.log"
done
echo "[$(date '+%Y-%m-%d %H:%M:%S')] docker_prepare complete"
fi
if [ "$EVAL_PREPARE" = true ]; then
echo "Waiting for all the instances to be prepared......You can use the following command to find the instances that are ready to be eval:"
echo "bash find_prepared.sh $model_name $dataset_name_short"
fi
############################
# Phase 3: start eval
############################
if [ "$GENERATE" = true ]; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Phase 3 (generate) starting"
conda activate cc_eval
CONDA_INIT='if [ -f "$HOME/anaconda3/etc/profile.d/conda.sh" ]; then . "$HOME/anaconda3/etc/profile.d/conda.sh"; elif [ -f "$HOME/miniconda3/etc/profile.d/conda.sh" ]; then . "$HOME/miniconda3/etc/profile.d/conda.sh"; fi'
main_cmd="$CONDA_INIT; conda activate cc_eval; python dispatcher/main.py \
--model-name $model_name \
--n-windows $generate_num_threads \
--start-index $start_index \
--end-index $end_index"
tmux new-session -d -s cc_session -n main bash -lc "${main_cmd} ; exec bash"
sleep 2.5
# Loop from 0 to generate_threads - 1
for i in $(seq 1 $((generate_num_threads))); do
tmux new-window -t cc_session -n "win$i" bash -lc "$CONDA_INIT; conda activate cc_eval; python dispatcher/client.py $i; exec bash"
done
tmux attach -t cc_session:0
tmux kill-server
fi
############################
# Phase 4: get results
############################
if [ "$GET_RESULTS" = true ]; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Phase 4 (get results) starting"
for instance_id in $(ls results/${model_name}/${dataset_name_short}/playground); do
repo_name=$(echo $instance_id | sed -E 's/.*__(.*)-.*/\1/')
echo "repo_name: $repo_name"
if [ "$(docker ps -q -f name=ccr_${instance_id}_${model_name})" ]; then
docker exec -w /testbed ccr_${instance_id}_${model_name} bash -c "
git diff > /workspace/diff_output.txt;
if [ ! -s /workspace/diff_output.txt ]; then
rm /workspace/diff_output.txt;
fi"
else
echo "Container ccr_${instance_id}_${model_name} does not exist."
fi
done
log_file="results/${model_name}/${dataset_name_short}/get_results/get_results_${timestamp}.log"
mkdir -p "results/${model_name}/${dataset_name_short}/get_results"
touch "${log_file}"
echo "You can check the get_results log file: $(pwd)/${log_file}"
python get_results.py \
--model_name $model_name \
--dataset_name_short $dataset_name_short > $log_file 2>&1
echo "All available patches are saved to $(pwd)/results/${model_name}/${dataset_name_short}/all_preds.jsonl"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Phase 4 (get results) complete"
fi
############################
# Phase 5: run final evaluation
############################
if [ "$FINAL_EVAL" = true ]; then
cd $eval_path
conda activate $eval_conda
echo "Current conda environment: $CONDA_DEFAULT_ENV"
start_time=$(date +%s)
predictions_path="../results/${model_name}/${dataset_name_short}/all_preds.jsonl"
errors_path="../results/${model_name}/${dataset_name_short}/eval_errors.jsonl"
result_path="../results/${model_name}/${dataset_name_short}/final_result.jsonl"
run_id="evaluation_${timestamp}"
mkdir -p "$(pwd)/../results/${model_name}/${dataset_name_short}/final_eval"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Phase 5 (run final evaluation) starting – log → tail -f $(pwd)/../results/${model_name}/${dataset_name_short}/final_eval/evaluation_${timestamp}.log"
log_file="$(pwd)/../results/${model_name}/${dataset_name_short}/final_eval/evaluation_${timestamp}.log"
touch "${log_file}"
{
stdbuf -oL -eL python -m swebench.harness.run_evaluation \
--dataset_name ${dataset_name} \
--predictions_path ${predictions_path} \
--max_workers ${execute_num_threads} \
--run_id ${run_id}_${num_lines}
} >> "${log_file}" 2>&1
{
stdbuf -oL -eL python next_step_eval.py \
--preds_file ${predictions_path} \
--errors_file ${errors_path} \
--output_file ${model_name}.${run_id}_${num_lines}.json \
--result_file ${result_path}
} >> "${log_file}" 2>&1
cd ..
bash stop_all_instances.sh ${model_name} -y
fi
end_time=$(date +%s)
elapsed=$((end_time - start_time))
echo "[✓] Full Process Complete – elapsed: ${elapsed}s ($(date -ud "@$elapsed" +'%Hh %Mm %Ss'))"Editor is loading...
Leave a Comment