+ set -Eeuo pipefail + ROOT=/home/jiachen/TRELLIS + EXP_ROOT=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview + TOOLKIT=/home/jiachen/.config/superpowers/worktrees/TRELLIS.2/trellis2-multiview-input/data_toolkit + CONDA=/home/jiachen/miniconda3/bin/conda + PY=/home/jiachen/miniconda3/envs/trellis2/bin/python + PROGRESS=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py + NOTIFY=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/notify_bark_experiment.py + SOURCE_ROOT=/data/jiachen/trellis2_vggt_toys4k + DATA_ROOT=/data/jiachen/trellis2_vggt_toys4k_25views + VAL_ROOT=/data/jiachen/trellis2_vggt_toys4k_25views/val64 + LOG_ROOT=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar + GPUS_CSV=4,5,6,7 + WORLD_SIZE=4 + MAX_WORKERS=1 + mkdir -p /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar + trap on_error ERR + main + setup_val_root + local source=/data/jiachen/trellis2_vggt_toys4k/val64 + mkdir -p /data/jiachen/trellis2_vggt_toys4k_25views/val64 + ln -sfn /data/jiachen/trellis2_vggt_toys4k/val64/raw /data/jiachen/trellis2_vggt_toys4k_25views/val64/raw + ln -sfn /data/jiachen/trellis2_vggt_toys4k/val64/metadata.csv /data/jiachen/trellis2_vggt_toys4k_25views/val64/metadata.csv + for name in mesh_dumps asset_stats dual_grid_1024 shape_latents ss_latents renders + '[' -e /data/jiachen/trellis2_vggt_toys4k/val64/mesh_dumps ']' + ln -sfn /data/jiachen/trellis2_vggt_toys4k/val64/mesh_dumps /data/jiachen/trellis2_vggt_toys4k_25views/val64/mesh_dumps + for name in mesh_dumps asset_stats dual_grid_1024 shape_latents ss_latents renders + '[' -e /data/jiachen/trellis2_vggt_toys4k/val64/asset_stats ']' + ln -sfn /data/jiachen/trellis2_vggt_toys4k/val64/asset_stats /data/jiachen/trellis2_vggt_toys4k_25views/val64/asset_stats + for name in mesh_dumps asset_stats dual_grid_1024 shape_latents ss_latents renders + '[' -e /data/jiachen/trellis2_vggt_toys4k/val64/dual_grid_1024 ']' + ln -sfn /data/jiachen/trellis2_vggt_toys4k/val64/dual_grid_1024 /data/jiachen/trellis2_vggt_toys4k_25views/val64/dual_grid_1024 + for name in mesh_dumps asset_stats dual_grid_1024 shape_latents ss_latents renders + '[' -e /data/jiachen/trellis2_vggt_toys4k/val64/shape_latents ']' + ln -sfn /data/jiachen/trellis2_vggt_toys4k/val64/shape_latents /data/jiachen/trellis2_vggt_toys4k_25views/val64/shape_latents + for name in mesh_dumps asset_stats dual_grid_1024 shape_latents ss_latents renders + '[' -e /data/jiachen/trellis2_vggt_toys4k/val64/ss_latents ']' + ln -sfn /data/jiachen/trellis2_vggt_toys4k/val64/ss_latents /data/jiachen/trellis2_vggt_toys4k_25views/val64/ss_latents + for name in mesh_dumps asset_stats dual_grid_1024 shape_latents ss_latents renders + '[' -e /data/jiachen/trellis2_vggt_toys4k/val64/renders ']' + ln -sfn /data/jiachen/trellis2_vggt_toys4k/val64/renders /data/jiachen/trellis2_vggt_toys4k_25views/val64/renders + progress_start --name 'Val64 25-view sidecar preprocessing' --goal '提前生成 val64 的 25-view renders_cond 和 VGGT/DINO feature cache，减少 train512 后的串行等待。' --dataset 'Toys4K val64 25-view correction' --cases 64 --gpu 4,5,6,7 --status running --progress 'starting low-concurrency render_cond25' --config 'render_cond.py --num_cond_views 25 --world_size 4 --max_workers 1; feature cache handled by incremental 4-GPU cacher; W&B not applicable for preprocessing.' --output /data/jiachen/trellis2_vggt_toys4k_25views/val64 + /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py start --id val64_25view_sidecar --name 'Val64 25-view sidecar preprocessing' --goal '提前生成 val64 的 25-view renders_cond 和 VGGT/DINO feature cache，减少 train512 后的串行等待。' --dataset 'Toys4K val64 25-view correction' --cases 64 --gpu 4,5,6,7 --status running --progress 'starting low-concurrency render_cond25' --config 'render_cond.py --num_cond_views 25 --world_size 4 --max_workers 1; feature cache handled by incremental 4-GPU cacher; W&B not applicable for preprocessing.' --output /data/jiachen/trellis2_vggt_toys4k_25views/val64 /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/report/a80_experiment_progress.html + local renders features ++ count_transforms ++ '[' '!' -d /data/jiachen/trellis2_vggt_toys4k_25views/val64/renders_cond ']' ++ find /data/jiachen/trellis2_vggt_toys4k_25views/val64/renders_cond -mindepth 2 -maxdepth 2 -type f -name transforms.json ++ wc -l ++ tr -d ' ' + renders=3 + '[' 3 -lt 64 ']' + log 'render_cond25 start gpus=4,5,6,7 world_size=4 max_workers=1 renders=3/64' ++ date -Is + tee -a /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/sidecar.log + echo '[2026-07-30T06:46:52+00:00] render_cond25 start gpus=4,5,6,7 world_size=4 max_workers=1 renders=3/64' [2026-07-30T06:46:52+00:00] render_cond25 start gpus=4,5,6,7 world_size=4 max_workers=1 renders=3/64 + progress_update --status running --progress 'render_cond25 running on GPUs 4,5,6,7: renders=3/64' --output /data/jiachen/trellis2_vggt_toys4k_25views/val64 + /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py update --id val64_25view_sidecar --status running --progress 'render_cond25 running on GPUs 4,5,6,7: renders=3/64' --output /data/jiachen/trellis2_vggt_toys4k_25views/val64 /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/report/a80_experiment_progress.html + run_sharded_render25 ++ date +%Y%m%d_%H%M%S + local log_dir=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652 + mkdir -p /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652 + rm -f '/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_*.exit' + (( rank = 0 )) + (( rank < WORLD_SIZE )) + local gpu ++ gpu_for_rank 0 ++ local rank=0 ++ IFS=, ++ read -ra gpus ++ '[' 4 -lt 4 ']' ++ echo 4 + gpu=4 + log 'render_cond25 rank=0/4 gpu=4 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_0.log' + set +e + export CUDA_VISIBLE_DEVICES=4 + CUDA_VISIBLE_DEVICES=4 + cd /home/jiachen/TRELLIS + /home/jiachen/miniconda3/bin/conda run --no-capture-output -n trellis2 python /home/jiachen/.config/superpowers/worktrees/TRELLIS.2/trellis2-multiview-input/data_toolkit/render_cond.py Toys4k --root /data/jiachen/trellis2_vggt_toys4k_25views/val64 --rank 0 --world_size 4 --num_cond_views 25 --max_workers 1 ++ date -Is + tee -a /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/sidecar.log + echo '[2026-07-30T06:46:52+00:00] render_cond25 rank=0/4 gpu=4 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_0.log' [2026-07-30T06:46:52+00:00] render_cond25 rank=0/4 gpu=4 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_0.log + (( rank++ )) + (( rank < WORLD_SIZE )) + local gpu ++ gpu_for_rank 1 ++ local rank=1 ++ IFS=, ++ read -ra gpus ++ '[' 4 -lt 4 ']' ++ echo 5 + gpu=5 + log 'render_cond25 rank=1/4 gpu=5 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_1.log' + tee -a /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/sidecar.log + set +e + export CUDA_VISIBLE_DEVICES=5 + CUDA_VISIBLE_DEVICES=5 + cd /home/jiachen/TRELLIS + /home/jiachen/miniconda3/bin/conda run --no-capture-output -n trellis2 python /home/jiachen/.config/superpowers/worktrees/TRELLIS.2/trellis2-multiview-input/data_toolkit/render_cond.py Toys4k --root /data/jiachen/trellis2_vggt_toys4k_25views/val64 --rank 1 --world_size 4 --num_cond_views 25 --max_workers 1 ++ date -Is + echo '[2026-07-30T06:46:52+00:00] render_cond25 rank=1/4 gpu=5 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_1.log' [2026-07-30T06:46:52+00:00] render_cond25 rank=1/4 gpu=5 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_1.log + (( rank++ )) + (( rank < WORLD_SIZE )) + local gpu ++ gpu_for_rank 2 ++ local rank=2 ++ IFS=, ++ read -ra gpus ++ '[' 4 -lt 4 ']' ++ echo 6 + gpu=6 + log 'render_cond25 rank=2/4 gpu=6 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_2.log' + tee -a /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/sidecar.log ++ date -Is + set +e + export CUDA_VISIBLE_DEVICES=6 + CUDA_VISIBLE_DEVICES=6 + cd /home/jiachen/TRELLIS + /home/jiachen/miniconda3/bin/conda run --no-capture-output -n trellis2 python /home/jiachen/.config/superpowers/worktrees/TRELLIS.2/trellis2-multiview-input/data_toolkit/render_cond.py Toys4k --root /data/jiachen/trellis2_vggt_toys4k_25views/val64 --rank 2 --world_size 4 --num_cond_views 25 --max_workers 1 + echo '[2026-07-30T06:46:52+00:00] render_cond25 rank=2/4 gpu=6 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_2.log' [2026-07-30T06:46:52+00:00] render_cond25 rank=2/4 gpu=6 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_2.log + (( rank++ )) + (( rank < WORLD_SIZE )) + local gpu ++ gpu_for_rank 3 ++ local rank=3 ++ IFS=, ++ read -ra gpus ++ '[' 4 -lt 4 ']' ++ echo 7 + gpu=7 + log 'render_cond25 rank=3/4 gpu=7 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_3.log' + set +e + export CUDA_VISIBLE_DEVICES=7 + CUDA_VISIBLE_DEVICES=7 + cd /home/jiachen/TRELLIS + /home/jiachen/miniconda3/bin/conda run --no-capture-output -n trellis2 python /home/jiachen/.config/superpowers/worktrees/TRELLIS.2/trellis2-multiview-input/data_toolkit/render_cond.py Toys4k --root /data/jiachen/trellis2_vggt_toys4k_25views/val64 --rank 3 --world_size 4 --num_cond_views 25 --max_workers 1 + tee -a /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/sidecar.log ++ date -Is + echo '[2026-07-30T06:46:52+00:00] render_cond25 rank=3/4 gpu=7 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_3.log' [2026-07-30T06:46:52+00:00] render_cond25 rank=3/4 gpu=7 log=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_3.log + (( rank++ )) + (( rank < WORLD_SIZE )) + wait + echo 0 + echo 0 + echo 0 + echo 0 + assert_sharded_success /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652 render_cond25 + local log_dir=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652 + local stage=render_cond25 + local failed=0 + (( rank = 0 )) + (( rank < WORLD_SIZE )) + local exit_file=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_0.exit + '[' '!' -f /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_0.exit ']' + local exit_code ++ cat /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_0.exit + exit_code=0 + '[' 0 '!=' 0 ']' + (( rank++ )) + (( rank < WORLD_SIZE )) + local exit_file=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_1.exit + '[' '!' -f /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_1.exit ']' + local exit_code ++ cat /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_1.exit + exit_code=0 + '[' 0 '!=' 0 ']' + (( rank++ )) + (( rank < WORLD_SIZE )) + local exit_file=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_2.exit + '[' '!' -f /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_2.exit ']' + local exit_code ++ cat /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_2.exit + exit_code=0 + '[' 0 '!=' 0 ']' + (( rank++ )) + (( rank < WORLD_SIZE )) + local exit_file=/home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_3.exit + '[' '!' -f /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_3.exit ']' + local exit_code ++ cat /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/render_cond25_20260730_064652/rank_3.exit + exit_code=0 + '[' 0 '!=' 0 ']' + (( rank++ )) + (( rank < WORLD_SIZE )) + '[' 0 -ne 0 ']' ++ count_transforms ++ '[' '!' -d /data/jiachen/trellis2_vggt_toys4k_25views/val64/renders_cond ']' ++ wc -l ++ tr -d ' ' ++ find /data/jiachen/trellis2_vggt_toys4k_25views/val64/renders_cond -mindepth 2 -maxdepth 2 -type f -name transforms.json + renders=64 ++ count_features ++ '[' '!' -d /data/jiachen/trellis2_vggt_toys4k_25views/val64/features ']' ++ find /data/jiachen/trellis2_vggt_toys4k_25views/val64/features -maxdepth 1 -type f -name '*.pt' ++ wc -l ++ tr -d ' ' + features=61 + log 'render stage done renders=64/64 features=61/64' ++ date -Is + tee -a /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/sidecar.log + echo '[2026-07-30T08:48:32+00:00] render stage done renders=64/64 features=61/64' [2026-07-30T08:48:32+00:00] render stage done renders=64/64 features=61/64 + progress_update --status running --progress 'render_cond25 done; feature cache pending: renders=64/64 features=61/64' --output /data/jiachen/trellis2_vggt_toys4k_25views/val64 + /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py update --id val64_25view_sidecar --status running --progress 'render_cond25 done; feature cache pending: renders=64/64 features=61/64' --output /data/jiachen/trellis2_vggt_toys4k_25views/val64 /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/report/a80_experiment_progress.html + cache_running_for_val + pgrep -af 'cache_toys4k_vggt_features.py .*--root /data/jiachen/trellis2_vggt_toys4k_25views/val64' ++ count_features ++ '[' '!' -d /data/jiachen/trellis2_vggt_toys4k_25views/val64/features ']' ++ wc -l ++ tr -d ' ' ++ find /data/jiachen/trellis2_vggt_toys4k_25views/val64/features -maxdepth 1 -type f -name '*.pt' + features=61 + '[' 61 -lt 64 ']' + local gpu ++ gpu_for_rank 0 ++ local rank=0 ++ IFS=, ++ read -ra gpus ++ '[' 4 -lt 4 ']' ++ echo 4 + gpu=4 + log 'feature cache start gpu=4 features=61/64' ++ date -Is + tee -a /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/sidecar.log + echo '[2026-07-30T08:48:32+00:00] feature cache start gpu=4 features=61/64' [2026-07-30T08:48:32+00:00] feature cache start gpu=4 features=61/64 + progress_update --status running --progress 'VGGT/DINO feature cache running on GPU4: features=61/64' --output /data/jiachen/trellis2_vggt_toys4k_25views/val64 + /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/a80_progress.py update --id val64_25view_sidecar --status running --progress 'VGGT/DINO feature cache running on GPU4: features=61/64' --output /data/jiachen/trellis2_vggt_toys4k_25views/val64 /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/report/a80_experiment_progress.html + export CUDA_VISIBLE_DEVICES=4 + CUDA_VISIBLE_DEVICES=4 + cd /home/jiachen/TRELLIS + /home/jiachen/miniconda3/envs/trellis2/bin/python /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/cache_toys4k_vggt_features.py --root /data/jiachen/trellis2_vggt_toys4k_25views/val64 --render-root /data/jiachen/trellis2_vggt_toys4k_25views/val64/renders_cond --output /data/jiachen/trellis2_vggt_toys4k_25views/val64/features --device cuda:0 --rank 0 --world-size 1 + tee -a /home/jiachen/TRELLIS/experiments/nonorthogonal_gpt_orthoview/outputs/logs/a80_25view_val64_sidecar/feature_cache.log Loading weights: 0%| | 0/415 [00:00