#!/usr/bin/env bash
set +e
cd /home/jiachen/TRELLIS
export CUDA_VISIBLE_DEVICES=7
/home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py update --id train512_state_anchor_statew025 --status running --progress running --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_state_anchor_statew025 --total-steps 1000
cmd=( /home/jiachen/miniconda3/bin/conda run --no-capture-output -n trellis2 python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/train_trellis2_vggt_state_anchor.py --root /data/jiachen/trellis2_vggt_toys4k/train512 --feature-root /data/jiachen/trellis2_vggt_toys4k/train512/features --ss-root /data/jiachen/trellis2_vggt_toys4k/train512/ss_latents/ss_enc_conv3d_16l8_fp16_64 --device cuda:0 --steps 1000 --accumulation 1 --checkpoint-steps 1\,100\,250\,500\,1000 --state-rel-cap 0.10 --state-loss-weight 0.25 --wandb-project trellis2-vggt-a80 --wandb-run-name a80_train512_state_anchor_statew025_gpu7 --wandb-group a80_train512 --wandb-tag train512 --wandb-tag state-anchor --wandb-tag statew025 --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_state_anchor_statew025 )
"${cmd[@]}" &
train_pid=$!
while kill -0 "$train_pid" 2>/dev/null; do
  /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py update --id train512_state_anchor_statew025 --status running --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_state_anchor_statew025 --total-steps 1000 --from-train-jsonl >/dev/null 2>&1 || true
  sleep 60
done
wait "$train_pid"
exit_code=$?
status="completed"
bark_status="完成"
conclusion="训练完成；最终指标见 train.jsonl / W&B。"
if [ "$exit_code" -ne 0 ]; then status="failed"; bark_status="失败"; conclusion="训练异常退出；请检查 logs/nohup.log。"; fi
/home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py finish --id train512_state_anchor_statew025 --status "$status" --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_state_anchor_statew025 --total-steps 1000 --from-train-jsonl --conclusion "$conclusion"
/home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/notify_bark_experiment.py --experiment a80_train512_state_anchor_statew025_gpu7 --status "$bark_status" --description Toys4K\ train512；state-anchor\ controller；state_loss_weight=0.25；1000\ steps；较弱状态锚定约束对照。 --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_state_anchor_statew025 --exit-code "$exit_code"
exit "$exit_code"