#!/usr/bin/env bash
set +e
cd /home/jiachen/TRELLIS
export CUDA_VISIBLE_DEVICES=6
/home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py start --id train512_25view_state_anchor_cap020 --name Train512\ 25-view\ state-anchor\ cap=0.20 --goal 25-view\ correction\ train512\ state-anchor\ ablation\ with\ random\ 2/4/6/8\ views. --dataset Toys4K\ train512\ 25-view\ correction --cases 512 --gpu 6 --status running --progress running --config state_rel_cap=0.20\;\ state_loss_weight=1.0\;\ W\&B\ group=a80_train512_25view_ablation --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_25view_state_anchor_cap020 --total-steps 1000
cmd=(/home/jiachen/miniconda3/bin/conda run --no-capture-output -n trellis2 python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/train_trellis2_vggt_state_anchor.py --root /data/jiachen/trellis2_vggt_toys4k_25views/train512 --feature-root /data/jiachen/trellis2_vggt_toys4k_25views/train512/features --render-root /data/jiachen/trellis2_vggt_toys4k_25views/train512/renders_cond --ss-root /data/jiachen/trellis2_vggt_toys4k_25views/train512/ss_latents/ss_enc_conv3d_16l8_fp16_64 --device cuda:0 --steps 1000 --accumulation 1 --checkpoint-steps 1,100,250,500,1000 --state-rel-cap 0.20 --state-loss-weight 1.0 --seed 20260732 --wandb-project trellis2-vggt-a80 --wandb-entity tycr --wandb-run-name train512_25view_state_anchor_cap020_gpu6 --wandb-group a80_train512_25view_ablation --wandb-tag train512 --wandb-tag 25view --wandb-tag state-anchor --wandb-tag ablation --wandb-notes 25-view\ correction\ run:\ random\ 2/4/6/8\ views\ sampled\ from\ 25\ renders_cond\;\ replacing\ the\ audited\ 16-view\ mismatch. --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_25view_state_anchor_cap020)
"${cmd[@]}" &
train_pid=$!
while kill -0 "$train_pid" 2>/dev/null; do
  /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py update --id train512_25view_state_anchor_cap020 --status running --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_25view_state_anchor_cap020 --total-steps 1000 --from-train-jsonl >/dev/null 2>&1 || true
  sleep 60
done
wait "$train_pid"
exit_code=$?
status=completed
bark_status=完成
conclusion="25-view ablation training completed; metrics are in train.jsonl and W&B."
if [ "$exit_code" -ne 0 ]; then status=failed; bark_status=失败; conclusion="25-view ablation training exited abnormally; check logs/nohup.log."; fi
/home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py finish --id train512_25view_state_anchor_cap020 --status "$status" --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_25view_state_anchor_cap020 --total-steps 1000 --from-train-jsonl --conclusion "$conclusion"
/home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/notify_bark_experiment.py --experiment train512_25view_state_anchor_cap020 --status "$bark_status" --description Toys4K\ train512\ 25-view\ correction；state-anchor\ adapter；state_rel_cap=0.20；state_loss_weight=1.0；1000\ steps；W\&B\ group=a80_train512_25view_ablation。 --output /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_25view_state_anchor_cap020 --exit-code "$exit_code"
if [ "$exit_code" -eq 0 ]; then
  eval_id=train512_25view_state_anchor_cap020_val64_eval
  eval_out=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_25view_state_anchor_cap020/val64_eval
  /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py start --id "$eval_id" --name Train512\ 25-view\ state-anchor\ cap=0.20\ val64\ eval --goal 固定\ val64\ 验证集评估\ 25-view\ state-anchor\ checkpoint\ 的\ loss/flow/state\ 指标。 --dataset Toys4K\ val64\ 25-view\ correction --cases 64 --gpu 6 --status running --progress running --config evaluate_state_anchor_val64.py\;\ state_rel_cap=0.20\;\ state_loss_weight=1.0\;\ W\&B\ group=a80_train512_25view_val64_eval --output "$eval_out"
  /home/jiachen/miniconda3/bin/conda run --no-capture-output -n trellis2 python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/evaluate_state_anchor_val64.py --root /data/jiachen/trellis2_vggt_toys4k_25views/val64 --feature-root /data/jiachen/trellis2_vggt_toys4k_25views/val64/features --render-root /data/jiachen/trellis2_vggt_toys4k_25views/val64/renders_cond --ss-root /data/jiachen/trellis2_vggt_toys4k_25views/val64/ss_latents/ss_enc_conv3d_16l8_fp16_64 --adapter-checkpoint /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/data_survey/vggt_spatial_adapter/train512_25view_state_anchor_cap020/adapter_step_1000.pt --output "$eval_out" --device cuda:0 --seed 20260732 --state-rel-cap 0.20 --state-loss-weight 1.0 --wandb-project trellis2-vggt-a80 --wandb-entity tycr --wandb-run-name train512_25view_state_anchor_cap020_val64_eval_gpu6 --wandb-group a80_train512_25view_val64_eval --wandb-tag val64 --wandb-tag 25view --wandb-tag state-anchor --wandb-tag ablation --wandb-notes Val64\ fixed-seed\ loss\ evaluation\ for\ the\ corresponding\ train512\ 25-view\ state-anchor\ ablation.
  eval_exit=$?
  eval_status=completed
  eval_conclusion="val64 evaluation completed; metrics are in val.jsonl/summary.json and W&B."
  if [ "$eval_exit" -ne 0 ]; then eval_status=failed; eval_conclusion="val64 evaluation exited abnormally; check training nohup.log."; fi
  /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py finish --id "$eval_id" --status "$eval_status" --output "$eval_out" --from-val-summary --conclusion "$eval_conclusion"
fi
exit "$exit_code"